Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
574c2ce794 | ||
|
|
7f5541eb5c | ||
|
|
7b49241c38 | ||
|
|
da716b2e3d | ||
|
|
a144fae8b2 | ||
|
|
0d90b82287 | ||
|
|
567c23e10f | ||
|
|
16970b7d37 | ||
|
|
8593cd362a | ||
|
|
2dbf96da81 | ||
|
|
83c7797187 | ||
|
|
6606deafa5 | ||
|
|
19b0d59159 | ||
|
|
9ccb3fa2a3 | ||
|
|
48533df523 | ||
|
|
1ae1137005 | ||
|
|
d5f2832b2f | ||
|
|
d0230b4893 | ||
|
|
a6cd53b97c | ||
|
|
7a28171a72 | ||
|
|
150bcb1116 | ||
|
|
6fe9c5bf00 | ||
|
|
3a86fc6680 | ||
|
|
2805cbb331 | ||
|
|
68d7a7014c | ||
|
|
883fa267a5 | ||
|
|
5f2c3ac965 | ||
|
|
bf8e32ebd2 | ||
|
|
5971be5bef | ||
|
|
677d76a3e4 | ||
|
|
6ed3207cb7 | ||
|
|
be6448b74f | ||
|
|
b707ee4844 | ||
|
|
0e9888be3b | ||
|
|
68b56b7858 | ||
|
|
d32697f611 | ||
|
|
3c54b2bcd7 | ||
|
|
899b182811 | ||
|
|
0dfb5e008d | ||
|
|
abf94357eb | ||
|
|
36aded3830 | ||
|
|
d7ebf50602 | ||
|
|
523c4c86a9 | ||
|
|
74cccdbfc7 | ||
|
|
70471eebfa | ||
|
|
6339224980 | ||
|
|
fe97f7a119 | ||
|
|
f8ed605da2 | ||
|
|
e38ef2af1f | ||
|
|
5d8e627d02 | ||
|
|
c9d4ad5450 | ||
|
|
31586bf588 | ||
|
|
974d3c83e3 | ||
|
|
8dc8ca382b | ||
|
|
1dbb250413 | ||
|
|
9f9ec692c7 | ||
|
|
2c1b54c76c | ||
|
|
7314494fb9 | ||
|
|
e74a5e8ef3 | ||
|
|
b9af999739 | ||
|
|
2de155f90f | ||
|
|
ab3db2e626 | ||
|
|
f9d26b04d8 | ||
|
|
24e0673198 | ||
|
|
833261801f | ||
|
|
6c8cbc1b9e | ||
|
|
6ae4e68d1c | ||
|
|
d22a2dee33 | ||
|
|
0f1ed589a5 | ||
|
|
b032e99dd1 | ||
|
|
fd7811f38d | ||
|
|
071db3a73d | ||
|
|
950c122850 | ||
|
|
e0386ea9dc | ||
|
|
6c3eee0f0e | ||
|
|
4c23add193 | ||
|
|
248d1a18c7 | ||
|
|
17d1b85423 | ||
|
|
b0deb63129 | ||
|
|
ebdb33bd11 | ||
|
|
69ccbda168 | ||
|
|
a38643cca6 | ||
|
|
712b19605e | ||
|
|
8a5f3842fc | ||
|
|
db3e578bcd | ||
|
|
7e22adff7c | ||
|
|
76fbaac658 | ||
|
|
c4fac29ecb | ||
|
|
33392b3351 | ||
|
|
0613cf2d59 | ||
|
|
a930ca0631 | ||
|
|
63a7271008 | ||
|
|
5ca0ebcbb2 | ||
|
|
2c84b7a7dd | ||
|
|
71a0c704bd | ||
|
|
d30e7cd29e | ||
|
|
aeb7c6fe76 | ||
|
|
134c20da93 | ||
|
|
893b4fc3ca | ||
|
|
9b27e81940 | ||
|
|
72c0acf269 | ||
|
|
7be53ad692 | ||
|
|
16e596ec87 | ||
|
|
3fdcb80960 | ||
|
|
26a3221bac | ||
|
|
eb16cb0aa8 | ||
|
|
b5340505e5 | ||
|
|
4bb8129bb7 | ||
|
|
7f838d3a31 | ||
|
|
c186c31bf7 | ||
|
|
86fb99a37c | ||
|
|
6c52450858 | ||
|
|
fb4e94b04a | ||
|
|
248baf58b4 | ||
|
|
0845d65f4e | ||
|
|
dcc5415648 | ||
|
|
3c38a88b22 | ||
|
|
187b7670b1 | ||
|
|
049c17ff08 | ||
|
|
8de97bb441 | ||
|
|
97c0092923 | ||
|
|
e65e1fb5a2 | ||
|
|
20dd1e55f5 | ||
|
|
6aa709e34e | ||
|
|
e73f121c08 | ||
|
|
3b9d6c4f8e | ||
|
|
69b60c19c3 | ||
|
|
04278cb1ab | ||
|
|
e8dec5b440 | ||
|
|
75f82b107e | ||
|
|
0e90c4b611 | ||
|
|
fb66b68bc8 | ||
|
|
c601677386 | ||
|
|
1fbd1cd04e | ||
|
|
4a71b91fbf | ||
|
|
b8b0d8b1ac | ||
|
|
8ef73d3985 | ||
|
|
05e556a5e3 | ||
|
|
d1da42bea8 | ||
|
|
90685e8357 | ||
|
|
a471d5d167 | ||
|
|
16122299e4 | ||
|
|
9f18da4bae | ||
|
|
1abce52867 | ||
|
|
072e2b65d3 | ||
|
|
f6d11e5467 | ||
|
|
458bddcb65 | ||
|
|
5fad43ff69 | ||
|
|
6992d0d947 | ||
|
|
d1b0180437 | ||
|
|
ba59e126cd | ||
|
|
4e3467bb7e | ||
|
|
4d4c705eb9 | ||
|
|
ce7b3184c4 | ||
|
|
b71faddaac | ||
|
|
1328cb4cce | ||
|
|
946488bce9 | ||
|
|
7713881e71 | ||
|
|
26fcc48c5a | ||
|
|
bf12fb68a2 | ||
|
|
3a4c9ff4f9 | ||
|
|
6180a2917d | ||
|
|
014cbb4623 | ||
|
|
ee98251d79 | ||
|
|
e3d08ec773 | ||
|
|
f03e68e63e | ||
|
|
72090b45dd | ||
|
|
93f9622798 | ||
|
|
16ff4b4e2c | ||
|
|
c90cca4b20 | ||
|
|
98d83ea74f | ||
|
|
a234f92d2c | ||
|
|
8ba3575901 | ||
|
|
a857c83c59 | ||
|
|
a6128ac8a2 | ||
|
|
2693301ad0 | ||
|
|
f7c7cb9535 | ||
|
|
b36a5f9b6b | ||
|
|
4826eb9969 | ||
|
|
c34b150794 | ||
|
|
c4c7670a7c | ||
|
|
0918bac86e | ||
|
|
a60a921706 | ||
|
|
b3ee29b66d | ||
|
|
3a1a9b988c | ||
|
|
12f2bebc46 | ||
|
|
31a3812c8a | ||
|
|
70ec08e30b | ||
|
|
23e8f0d849 | ||
|
|
1cdfcd5b4e | ||
|
|
dc2786fbc0 | ||
|
|
7e39b0c4fe | ||
|
|
4af830c405 | ||
|
|
fe844581fd | ||
|
|
443ce6352b | ||
|
|
158c354a27 | ||
|
|
92d1230a90 | ||
|
|
d64916a061 | ||
|
|
20a0568253 | ||
|
|
5a1f42ee54 | ||
|
|
ab95f14e21 | ||
|
|
bae2676b69 | ||
|
|
ea6869a63e | ||
|
|
953327c150 | ||
|
|
e01ad6bc85 | ||
|
|
47a33009ff | ||
|
|
bc844013a1 | ||
|
|
2fd2319cdb | ||
|
|
a50657c472 | ||
|
|
ef5f6e06ba | ||
|
|
977cf9f506 | ||
|
|
ffae10d11f | ||
|
|
221107e76e | ||
|
|
e74c78b607 | ||
|
|
434c7b57b1 | ||
|
|
9dc4608a62 | ||
|
|
fc69366092 | ||
|
|
9f0b1f3e25 | ||
|
|
3f03a845ec | ||
|
|
bc28976e12 | ||
|
|
8f826098f9 | ||
|
|
0a765e6207 | ||
|
|
f83aa61923 | ||
|
|
f9ff3e5d14 | ||
|
|
b3fde9d78d | ||
|
|
0c999083cc | ||
|
|
561215db8a | ||
|
|
8dbb7f2c78 | ||
|
|
6f7f3bb316 | ||
|
|
8d507c4b73 | ||
|
|
2beb0eda94 | ||
|
|
b7e07b72ca | ||
|
|
fde8718c8c | ||
|
|
b2d1bddcfe | ||
|
|
dd1d1be994 | ||
|
|
a93957178a | ||
|
|
5026d0b2f3 | ||
|
|
cc8b47d882 | ||
|
|
db0b5961b3 | ||
|
|
cb6f7ada42 | ||
|
|
59464099b4 | ||
|
|
aed00b53fd | ||
|
|
cdb0f97cb4 | ||
|
|
33bcc8256c | ||
|
|
0a8d44f12f | ||
|
|
1b74594f5b | ||
|
|
b0718d295e | ||
|
|
9bb91b126d | ||
|
|
6dbb00d8d5 | ||
|
|
3a27a4509a | ||
|
|
1d438f5d23 | ||
|
|
3840f19869 | ||
|
|
ec0483b975 | ||
|
|
2fba1f417f | ||
|
|
5b419b55c5 | ||
|
|
b31379e51c | ||
|
|
8394dcb663 | ||
|
|
457a56ce04 | ||
|
|
7c8016d807 | ||
|
|
d704336f20 | ||
|
|
1b4230b06e | ||
|
|
88095af1cd | ||
|
|
e9efd9f68a | ||
|
|
7b32e2ff7c | ||
|
|
572777305e | ||
|
|
7e27a6e37d | ||
|
|
b98621d7ea | ||
|
|
bc5824b8bc | ||
|
|
a6b643bb34 | ||
|
|
488b976734 | ||
|
|
253912b52e | ||
|
|
0bf8416d9f | ||
|
|
2e921437d8 | ||
|
|
b6f9b60683 | ||
|
|
57430e9f5f | ||
|
|
1e13fbee8d | ||
|
|
3d8b9e2b66 | ||
|
|
64898d43f1 | ||
|
|
b2117fddab | ||
|
|
5658623f29 | ||
|
|
934682bd0f | ||
|
|
daecc582f3 | ||
|
|
3cb4012dbf | ||
|
|
f7bd44a4f5 | ||
|
|
52317b05bb | ||
|
|
83c0baa3cb | ||
|
|
340c5addab | ||
|
|
e3fd94dbf0 | ||
|
|
923fcfb9e9 | ||
|
|
0ae5f1a4c6 | ||
|
|
90e06abba2 | ||
|
|
6ecfc94ff1 | ||
|
|
60c4a727d3 | ||
|
|
9fbb9f2e98 | ||
|
|
f1feec66c0 | ||
|
|
73b9b983b1 | ||
|
|
588779e3a4 | ||
|
|
b7ae446059 | ||
|
|
db898307b5 | ||
|
|
7e3512cbf2 | ||
|
|
cdd65c6783 | ||
|
|
320ad717cd | ||
|
|
264e6744e1 | ||
|
|
64e2b79c07 | ||
|
|
9255bb5abe | ||
|
|
d7525dacc3 | ||
|
|
5cb07100f8 | ||
|
|
171c080d33 | ||
|
|
cfc3ec2a05 | ||
|
|
1392b8f83b | ||
|
|
8b265801df | ||
|
|
6c4fb8dab1 | ||
|
|
c1a9cd93ee | ||
|
|
0d4e83e167 | ||
|
|
1292eb345d | ||
|
|
f4c6dc90c9 | ||
|
|
8c8f5d8f58 | ||
|
|
b6d986cb40 | ||
|
|
6cc134c7c8 | ||
|
|
eee964cfa0 | ||
|
|
a03c94bfad | ||
|
|
f1f0917063 | ||
|
|
fe2476b5aa | ||
|
|
9c5fa8bf76 | ||
|
|
cd43a71172 | ||
|
|
e2c6de76f4 | ||
|
|
05aeadc9fe | ||
|
|
7153417372 | ||
|
|
fceb2ae0a2 | ||
|
|
0cef2fdd82 | ||
|
|
9c91b135bd | ||
|
|
e3b127192c | ||
|
|
ee1738da6e | ||
|
|
8ca2b75c80 | ||
|
|
dc81b6def1 | ||
|
|
22f5aaee70 | ||
|
|
4bd6569c9c | ||
|
|
dbf39baeb1 | ||
|
|
3adf3c26f2 | ||
|
|
fa0d13e630 | ||
|
|
227a82d6d3 | ||
|
|
65cfc1f63c | ||
|
|
e46f0e2444 | ||
|
|
00bc7995c4 | ||
|
|
2dcebe48b7 | ||
|
|
d12b6cddb3 | ||
|
|
699c057f2d | ||
|
|
3ce9f31f4b | ||
|
|
485d5d55e8 | ||
|
|
6ee9f2513b | ||
|
|
81a6e14a5c | ||
|
|
496ffe0096 | ||
|
|
216c760179 | ||
|
|
01eca3004f | ||
|
|
2f47a3758e | ||
|
|
73bcc1796f | ||
|
|
e03e41ad31 | ||
|
|
6cb30f8e05 | ||
|
|
d9b1be6214 | ||
|
|
5f2f06c269 | ||
|
|
e1dcdcf4db | ||
|
|
416472c5d8 | ||
|
|
196f3ba92d | ||
|
|
6feae93c58 | ||
|
|
203d30d4f7 | ||
|
|
d0a7ca841b | ||
|
|
2e6c848c39 | ||
|
|
18a0204105 | ||
|
|
01baa193d5 | ||
|
|
ba57af5502 | ||
|
|
8139f93ff1 | ||
|
|
4596d5c26f | ||
|
|
810a63782d | ||
|
|
ea83827d57 | ||
|
|
e19f8f3d95 | ||
|
|
7af501e0bc | ||
|
|
a692b6ff7d | ||
|
|
3364ac0976 | ||
|
|
cf1157f02a | ||
|
|
0ed56bbe1a | ||
|
|
213bdac55f | ||
|
|
6d8ab97ca9 | ||
|
|
fb34476cd7 | ||
|
|
07d353a947 | ||
|
|
3c5fb6490d | ||
|
|
a5d21081b9 | ||
|
|
ed98a3cc21 | ||
|
|
45c2cd2986 | ||
|
|
75f21950da | ||
|
|
3f704be85c | ||
|
|
67a15ea788 | ||
|
|
55bcd2dfee | ||
|
|
d9757fdae8 | ||
|
|
a4ae8d7c53 | ||
|
|
78078b4c9f | ||
|
|
1331e2bbe6 | ||
|
|
4c4d652f1b | ||
|
|
73bd76b631 | ||
|
|
59a0f9f602 | ||
|
|
1b1847bc31 | ||
|
|
21a569edcc | ||
|
|
2ddf1a08f6 | ||
|
|
8754c6fabe | ||
|
|
f049109a38 | ||
|
|
8db7c15aa1 | ||
|
|
90e0694d2e | ||
|
|
4748cbf654 | ||
|
|
74eb991983 | ||
|
|
e11bae2dda | ||
|
|
8462f56b13 | ||
|
|
0bec3c7294 | ||
|
|
2bb4052193 | ||
|
|
101a3f2b22 | ||
|
|
04a2bde8dd | ||
|
|
a1368b05da | ||
|
|
99ea7a26b0 | ||
|
|
8b3a457925 | ||
|
|
e02e2a249a | ||
|
|
3019a1c222 | ||
|
|
1f3a19c100 | ||
|
|
5eac4247e1 | ||
|
|
1ca82b9d52 | ||
|
|
21702ec43c | ||
|
|
d2083bc17e | ||
|
|
52edf880e0 | ||
|
|
1f45af991f | ||
|
|
8afdd0b2e9 | ||
|
|
99efc70ba3 | ||
|
|
64e306801d | ||
|
|
330db61b36 | ||
|
|
357763ff61 | ||
|
|
8a6e6b4d77 | ||
|
|
64913c4288 | ||
|
|
f01ad0e4bd | ||
|
|
d9fcc458dc | ||
|
|
24a11d3f21 | ||
|
|
234eaf8890 | ||
|
|
9e2e53147e | ||
|
|
fc41af5889 | ||
|
|
72ce0cd0b0 | ||
|
|
e4497c1327 | ||
|
|
a1bf832ea6 | ||
|
|
d4e76500fb | ||
|
|
f898bb3d0e | ||
|
|
ad13282494 | ||
|
|
73519f8c6a | ||
|
|
b2c95f35c7 | ||
|
|
0bfccb86ff | ||
|
|
5118240897 | ||
|
|
427fc179ac | ||
|
|
55bcc520e4 | ||
|
|
bc8251696e | ||
|
|
677ba25ccd | ||
|
|
95436b233c | ||
|
|
82871b8f6b | ||
|
|
9525b87224 | ||
|
|
f06098978c | ||
|
|
141491f2da | ||
|
|
38006f7d29 | ||
|
|
3664c25d98 | ||
|
|
1747c9d866 | ||
|
|
1066fbaa7c | ||
|
|
7f7961f4cc | ||
|
|
9d9916548b | ||
|
|
ada5b55f22 | ||
|
|
a2f28e7788 | ||
|
|
c6092a4c79 | ||
|
|
315a557019 | ||
|
|
20221e1186 | ||
|
|
50003d2600 | ||
|
|
ea6b2b98fb | ||
|
|
4890fdfb50 | ||
|
|
2f95c3a9d1 | ||
|
|
6039159ecc | ||
|
|
8e4d8345bd | ||
|
|
22df26eeb6 | ||
|
|
7b9ff3e8b5 | ||
|
|
1a5b2c08ee | ||
|
|
b6babb1c9d | ||
|
|
a88485c5a0 | ||
|
|
0cefa07e97 | ||
|
|
ca9d48d574 | ||
|
|
e10e1de92f | ||
|
|
bdf6887791 | ||
|
|
295f2c1115 | ||
|
|
78fcf6a44a | ||
|
|
7dcebf99e0 | ||
|
|
463cbde4b1 | ||
|
|
1adce8b227 | ||
|
|
673d259d7a | ||
|
|
d0b739a0b9 | ||
|
|
b4aa51444c | ||
|
|
9178ca701c | ||
|
|
7eb9f344f4 | ||
|
|
9f8d3e69ae | ||
|
|
e1130147b5 |
No files matched your search
@@ -0,0 +1,7 @@
|
||||
.git
|
||||
.tox
|
||||
__pycache__
|
||||
dist
|
||||
lbrynet.egg-info
|
||||
docs
|
||||
tests
|
||||
@@ -51,3 +51,24 @@ Tell us what happens instead
|
||||
|
||||
## Screenshots
|
||||
<!-- If a screenshot would help explain the bug, please include one or two here -->
|
||||
|
||||
|
||||
|
||||
## Internal Use
|
||||
|
||||
### Acceptance Criteria
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
### Definition of Done
|
||||
- [ ] Tested against acceptance criteria
|
||||
- [ ] Tested against the assumptions of user story
|
||||
- [ ] The project builds without errors
|
||||
- [ ] Unit tests are written and passing
|
||||
- [ ] Tests on devices/browsers listed in the issue have passed
|
||||
- [ ] QA performed & issues resolved
|
||||
- [ ] Refactoring completed
|
||||
- [ ] Any configuration or build changes documented
|
||||
- [ ] Documentation updated
|
||||
- [ ] Peer Code Review performed
|
||||
@@ -0,0 +1,38 @@
|
||||
## PR Checklist
|
||||
Please check all that apply to this PR using "x":
|
||||
|
||||
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
|
||||
- [ ] I have checked that this PR does not introduce a breaking change
|
||||
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
|
||||
|
||||
|
||||
## PR Type
|
||||
What kind of change does this PR introduce?
|
||||
|
||||
Why is this change necessary?
|
||||
|
||||
<!-- Please check all that apply to this PR using "x". -->
|
||||
|
||||
- [ ] Bugfix
|
||||
- [ ] Feature
|
||||
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
|
||||
- [ ] Code style update (formatting)
|
||||
- [ ] Refactoring (no functional changes)
|
||||
- [ ] Documentation changes
|
||||
- [ ] Other - Please describe:
|
||||
|
||||
## Fixes
|
||||
|
||||
Issue Number: N/A
|
||||
|
||||
|
||||
## What is the current behavior?
|
||||
|
||||
|
||||
## What is the new behavior?
|
||||
|
||||
|
||||
## Other information
|
||||
|
||||
|
||||
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS
|
||||
ignore=CVS,schema
|
||||
|
||||
# Add files or directories matching the regex patterns to the
|
||||
# blacklist. The regex matches against base names, not paths.
|
||||
@@ -62,7 +62,7 @@ confidence=
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once).You can also use "--disable=all" to
|
||||
# disable everything first and then reenable specific checks. For example, if
|
||||
# disable everything first and then re-enable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||
|
||||
@@ -9,44 +9,35 @@ jobs:
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid==2.0.4 aiohttp==3.4.4
|
||||
# newer astroid and aiohttp fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git
|
||||
- pip install git+https://github.com/lbryio/lbryschema.git
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
|
||||
- &tests
|
||||
stage: test
|
||||
name: "Unit Tests w/ Python 3.7"
|
||||
name: "Unit Tests"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git
|
||||
- pip install git+https://github.com/lbryio/lbryschema.git
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .[test]
|
||||
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.schema tests.unit.wallet tests.unit.components tests.unit.test_conf
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.test_cli
|
||||
#- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- <<: *tests
|
||||
name: "Unit Tests w/ Python 3.6"
|
||||
python: "3.6"
|
||||
|
||||
- <<: *tests
|
||||
name: "DHT Tests w/ Python 3.7"
|
||||
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
|
||||
|
||||
- <<: *tests
|
||||
name: "DHT Tests w/ Python 3.6"
|
||||
python: "3.6"
|
||||
name: "DHT Tests"
|
||||
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
|
||||
|
||||
- name: "Integration Tests"
|
||||
install:
|
||||
- pip install tox-travis coverage
|
||||
- pushd .. && git clone https://github.com/lbryio/electrumx.git --branch lbryumx && popd
|
||||
- pushd .. && git clone https://github.com/lbryio/orchstr8.git && popd
|
||||
- pushd .. && git clone https://github.com/lbryio/lbryschema.git && popd
|
||||
- pushd .. && git clone https://github.com/lbryio/lbryumx.git && cd lbryumx && git checkout afd34f323dd94c516108a65240f7d17aea8efe85 && cd .. && popd
|
||||
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
|
||||
script: tox
|
||||
after_success:
|
||||
@@ -59,32 +50,50 @@ jobs:
|
||||
services:
|
||||
- docker
|
||||
install:
|
||||
- docker pull cdrx/pyinstaller-windows:python3-32bit
|
||||
- docker pull lbry/pyinstaller34_32bits:py371
|
||||
script:
|
||||
- docker run -v "$(pwd):/src/lbry" cdrx/pyinstaller-windows:python3-32bit lbry/scripts/wine_build.sh
|
||||
- python scripts/set_build.py
|
||||
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
|
||||
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-windows.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
paths:
|
||||
- lbrynet-windows.zip
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})/win/
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- &build
|
||||
name: "Linux"
|
||||
python: "3.6"
|
||||
env: OS=linux
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git
|
||||
- pip3 install git+https://github.com/lbryio/lbryschema.git
|
||||
- python scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbrynet/cli.py
|
||||
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
|
||||
- ./dist/lbrynet --version
|
||||
env: OS=linux
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-${OS}.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
@@ -92,14 +101,27 @@ jobs:
|
||||
- lbrynet-${OS}.zip
|
||||
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})/${OS}/lbrynet
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- <<: *build
|
||||
name: "Mac"
|
||||
os: osx
|
||||
osx_image: xcode9.4
|
||||
osx_image: xcode8.3
|
||||
language: generic
|
||||
env: OS=mac
|
||||
cache: false
|
||||
before_install:
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
- pip3 install pyinstaller
|
||||
- git clone https://github.com/lbryio/torba.git --depth 1
|
||||
- sed -i -e "s/'plyvel',//" torba/setup.py
|
||||
- cd torba && pip3 install -e . && cd ..
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
|
||||
cache:
|
||||
directories:
|
||||
|
||||
@@ -7,17 +7,64 @@ regard to the json-rpc api. As we're currently pre-1.0 release, we
|
||||
can and probably will change functionality and break backwards compatibility
|
||||
at anytime.
|
||||
|
||||
## [Unreleased]
|
||||
Python 3 upgrade of the entire code base and switching to a brand new wallet
|
||||
implementation are the major changes in this release.
|
||||
|
||||
## [0.30.1] - 2018-12-09
|
||||
|
||||
### API
|
||||
* add `aioupnp_version` to `upnp` field in the response to `status`
|
||||
* fixed uncaught claim signature errors
|
||||
* `--blocking` support for `abandon` command (https://github.com/lbryio/lbry/pull/1618)
|
||||
* don't create SSL keys if SSL is disabled (https://github.com/lbryio/lbry/pull/1644)
|
||||
* added `account_send` command (https://github.com/lbryio/lbry/pull/1654)
|
||||
* fixes for claim send to address (https://github.com/lbryio/lbry/pull/1651)
|
||||
* better handle errors on resolve (https://github.com/lbryio/lbry/pull/1662)
|
||||
* fix `transaction_show` (https://github.com/lbryio/lbry/pull/1689)
|
||||
|
||||
### Schema
|
||||
* merged `lbryschema` repository into `lbrynet` (https://github.com/lbryio/lbry/pull/1444)
|
||||
* updated `protobuf` to `3.2.0` (https://github.com/lbryio/lbry/issues/1602)
|
||||
* use `types` repo for generating python protobufs (https://github.com/lbryio/lbry/pull/1605)
|
||||
|
||||
### Wallet
|
||||
* fixed resolving names with invalid channel signatures
|
||||
* fixed wallet sync issues
|
||||
* fixed wallet locking/encryption issues
|
||||
|
||||
### DHT
|
||||
* fixed retrying kademlia rpc calls (https://github.com/lbryio/lbry/pull/1495)
|
||||
* fixed `KeyError` in `iterativeFindValue`, this is used by `peer_list` and when finding peers to download blobs from (https://github.com/lbryio/lbry/issues/1638)
|
||||
* settings external ip address and ability to disable more DHT components (https://github.com/lbryio/lbry/pull/1604)
|
||||
* improved routing table bucket initialization (https://github.com/lbryio/lbry/pull/1633)
|
||||
|
||||
### Database
|
||||
* resolved too many variables in sqlite issue (https://github.com/lbryio/lbry/pull/1612)
|
||||
|
||||
### Misc
|
||||
* switch to Python 3.7
|
||||
* brought back icon for windows binary
|
||||
* using custom list of mime types with more types (https://github.com/lbryio/lbry/pull/1700)
|
||||
* add balance to startup report (https://github.com/lbryio/lbry/pull/1613)
|
||||
* drop `six` dependency (https://github.com/lbryio/lbry/pull/1660)
|
||||
* drop `mock` dependency (https://github.com/lbryio/lbry/pull/1661)
|
||||
|
||||
## [0.30.0] - 2018-10-22
|
||||
This release is the most ambitious upgrade to lbrynet yet; including a brand
|
||||
new wallet implementation, switch to Python 3 of the entire code base and
|
||||
many changes to the API such as pagination and a more consistent output from
|
||||
most commands.
|
||||
|
||||
|
||||
### Security
|
||||
* upgraded `cryptography` package.
|
||||
* switch to `coincurve` for private / public key generation (addresses)
|
||||
* support for connecting to daemon via SSL
|
||||
* improved workflow for encrypt/decrypt of wallet
|
||||
|
||||
### API
|
||||
* unified all command line executables into a single `lbrynet` executable.
|
||||
* added `use_https` configuration setting (off by default), if true the api server will generate a self signed ssl certificate when started and only handle api requests on the https address (default of `https://localhost:5279/lbryapi`)
|
||||
* added `use_https` configuration setting (off by default), if true the api
|
||||
server will generate a self signed ssl certificate when started and only
|
||||
handle api requests on the https address (default of `https://localhost:5279/lbryapi`)
|
||||
* deprecated `daemon_stop` command, use `stop` instead.
|
||||
* deprecated `wallet_balance` command, use `account_balance` instead.
|
||||
* deprecated `wallet_unlock` command, use `account_unlock` instead.
|
||||
@@ -27,7 +74,7 @@ implementation are the major changes in this release.
|
||||
* deprecated `wallet_list` command, use `address_list` instead.
|
||||
* deprecated `wallet_is_address_mine` command, use `address_is_mine` instead.
|
||||
* deprecated `wallet_public_key` command.
|
||||
* deprecated `wallet_new_address` command, use `address_generate` instead.
|
||||
* deprecated `wallet_new_address` command.
|
||||
* deprecated `wallet_unused_address` command, use `address_unused` instead.
|
||||
* deprecated `claim_renew` command.
|
||||
* added `account_list` command to list accounts including their balance.
|
||||
@@ -44,12 +91,14 @@ implementation are the major changes in this release.
|
||||
* added `account_max_address_gap` command to find large gaps of unused addresses.
|
||||
* added `address_list` command to list addresses.
|
||||
* added `address_is_mine` command to check if an address is one of your addresses.
|
||||
* added `address_public_key` command to get public key of an address.
|
||||
* added `address_generate` command to generate a new address.
|
||||
* added `address_unused` command to get existing or generate a new unused address.
|
||||
* added pagination support for `address_list`, `channel_list`, `claim_list_mine`,
|
||||
`transaction_list` and `utxo_list`.
|
||||
* added `upnp` field to `status` response
|
||||
* removed `send_amount_to_address` command previously marked as deprecated
|
||||
* removed `channel_list_mine` command previously marked as deprecated
|
||||
* removed `get_availability` command previously marked as deprecated
|
||||
* updated `txupnp` dependency to `aioupnp` (UPnP for asyncio)
|
||||
|
||||
### Wallet
|
||||
* changed to a new wallet implementation: [torba](https://github.com/lbryio/torba).
|
||||
@@ -68,11 +117,13 @@ implementation are the major changes in this release.
|
||||
* Extensive internal changes as a result of porting to Python 3.
|
||||
|
||||
### Reflector
|
||||
*
|
||||
* No major changes to reflector.
|
||||
|
||||
### Database
|
||||
*
|
||||
* No changes to existing storage module.
|
||||
|
||||
### Known Issues
|
||||
* Channel certificate import/export is currently not working.
|
||||
|
||||
## [0.21.2] - 2018-08-23
|
||||
### Fixed
|
||||
@@ -435,7 +486,7 @@ implementation are the major changes in this release.
|
||||
* Fixed https://github.com/lbryio/lbry/issues/923
|
||||
* Fixed concurrent reflects opening too many files
|
||||
* Fixed cases when reflecting would fail on error conditions
|
||||
* Fixed deadlocks from occuring during blob writes
|
||||
* Fixed deadlocks from occurring during blob writes
|
||||
* Fixed and updated`lbrynet.tests.dht`
|
||||
* Fixed redundant dht id
|
||||
* Fixed dht `ping` method
|
||||
@@ -822,7 +873,7 @@ implementation are the major changes in this release.
|
||||
* Removed check_pending logic from Daemon
|
||||
* Switched to txrequests so requests can use twisted event loop
|
||||
* Renamed API command file_seed to file_set_status
|
||||
* Dont add expected payment to wallet when payment rate is 0
|
||||
* Don't add expected payment to wallet when payment rate is 0
|
||||
### Fixed
|
||||
* Fix restart procedure in DaemonControl
|
||||
* Create download directory if it doesn't exist
|
||||
@@ -853,7 +904,7 @@ implementation are the major changes in this release.
|
||||
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
|
||||
* Remove unused API commands from daemon
|
||||
* Fix file filter `outpoint`
|
||||
* Made dictionary key names in API commmand outputs to be more consistent
|
||||
* Made dictionary key names in API command outputs to be more consistent
|
||||
### Added
|
||||
* Add file filters: `claim_id`, `outpoint`, and `rowid`
|
||||
* Make loggly logs less verbose
|
||||
@@ -905,7 +956,7 @@ implementation are the major changes in this release.
|
||||
* Include download url in version check
|
||||
|
||||
### Fixed
|
||||
* add misssing traceback to logging
|
||||
* add missing traceback to logging
|
||||
|
||||
## [0.8.3] - 2017-02-15
|
||||
### Fixed
|
||||
|
||||
@@ -1,70 +1,48 @@
|
||||
# Installing LBRY
|
||||
|
||||
The recommended way to install LBRY is to use an installer. We provide installers for all major operating systems. See the [README](README.md).
|
||||
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
|
||||
|
||||
These instructions are for installing from source code, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
|
||||
Here's a video walkthrough of this setup, which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch):
|
||||
[](https://spee.ch/967f99344308f1e90f0620d91b6c93e4dfb240e0/lbrynet-dev-setup.mp4)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before starting, you'll need to install some additional libraries depending on your operating system.
|
||||
Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred). Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
|
||||
|
||||
### Python
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
LBRY requires Python 2.7. Python 3 is not supported.
|
||||
### Virtualenv
|
||||
|
||||
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
|
||||
|
||||
### Windows
|
||||
|
||||
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
|
||||
|
||||
|
||||
### OS X Prerequisites
|
||||
### macOS
|
||||
|
||||
Install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install mpfr libmpc protobuf
|
||||
sudo easy_install pip
|
||||
sudo pip install virtualenv
|
||||
brew install python3 protobuf
|
||||
```
|
||||
|
||||
### Linux Prerequisites
|
||||
### Linux
|
||||
|
||||
On Ubuntu (we recommend 16.04), install the following:
|
||||
On Ubuntu (we recommend 18.04), install the following:
|
||||
|
||||
```
|
||||
sudo apt-get install build-essential python2.7 python2.7-dev python-pip git python-virtualenv libssl-dev libffi-dev python-protobuf
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
|
||||
If you're running another Linux flavor, install the equivalent of the above packages for your system.
|
||||
|
||||
### Windows Prerequisites
|
||||
|
||||
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
|
||||
|
||||
Add C:\MinGW\bin to the windows PATH.
|
||||
|
||||
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing:
|
||||
|
||||
```
|
||||
[build]
|
||||
compiler = mingw32
|
||||
```
|
||||
|
||||
If using virtualenv, which is recommended, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
|
||||
|
||||
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
|
||||
Currently, available binaries include:
|
||||
- Cython
|
||||
- Twisted
|
||||
- Zope.interface
|
||||
- pywin32
|
||||
- requests
|
||||
|
||||
Install the above binaries with `pip install *.whl`
|
||||
|
||||
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
|
||||
|
||||
Finally, you'll need [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9). Download the source and compile with MinGW by running `mingw32make.bat`. Then install the module by running `python setupmingw32.py install`.
|
||||
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -73,35 +51,27 @@ To install:
|
||||
```
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
virtualenv lbry-venv
|
||||
|
||||
virtualenv lbry-venv --python=python3.7
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
python --version # Python 3 is not supported. Make sure you're on Python 2.7
|
||||
|
||||
pip install -U -r requirements.txt
|
||||
pip install -U -r requirements_testing.txt
|
||||
pip install --editable .
|
||||
|
||||
python --version # Python 2 is not supported. Make sure you're on Python 3.7
|
||||
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
To start LBRY, run `lbrynet-daemon` in a terminal.
|
||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
|
||||
To verify your install, `which lbrynet-daemon` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
## Run the tests
|
||||
To run the unit tests from the repo directory:
|
||||
```
|
||||
trial --reactor=asyncio tests.unit
|
||||
```
|
||||
|
||||
## Making a Change
|
||||
## Usage
|
||||
|
||||
If you are not already inside the virtual environment, run:
|
||||
To start the API server:
|
||||
`lbrynet start`
|
||||
|
||||
```
|
||||
source lbry-venv/bin/activate
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```
|
||||
pip install --editable .
|
||||
```
|
||||
|
||||
This will install `lbrynet-daemon` in such a way that the changes you make to the code will be used as soon as you restart the daemon.
|
||||
|
||||
Happy hacking!
|
||||
@@ -1,28 +1,35 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized network. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, and rights and access rules. LBRY also provides a data network that consists of peers uploading and downloading data from other peers, possibly in exchange for payments, and a distributed hash table, used by peers to discover other peers.
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
This project aims to provide a daemon that can interact with the network via a json-rpc interface over HTTP.
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
The project is written in Python 3.7+ and extensively uses Twisted framework.
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
|
||||
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases, for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
|
||||
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
|
||||
|
||||
## Usage
|
||||
|
||||
Run `lbrynet start` to launch the daemon.
|
||||
Run `lbrynet start` to launch the API server.
|
||||
|
||||
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
|
||||
Our [quickstart guide](http://lbry.io/quickstart) provides a simple walkthrough and examples for learning.
|
||||
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
|
||||
|
||||
The full API is documented [here](https://lbryio.github.io/lbry/cli).
|
||||
With the daemon running, `lbrynet commands` will show you a list of commands.
|
||||
|
||||
The full API is documented [here](https://lbry.tech/api/sdk).
|
||||
|
||||
## Running from source
|
||||
|
||||
Installing from source is also relatively painless, full instructions are in [INSTALL.md](INSTALL.md)
|
||||
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
|
||||
|
||||
## Contributing
|
||||
|
||||
@@ -34,12 +41,16 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues. Our PGP key is here if you need it.
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@jackrobison](mailto:jack@lbry.io)
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
The documentation for the api can be found [here](https://lbry.io/api).
|
||||
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
|
||||
|
||||
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
|
||||
|
||||
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry/blob/master/example_daemon_settings.yml).
|
||||
@@ -100,7 +100,7 @@
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="/" title="LBRY" class="md-header-nav__button md-logo">
|
||||
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="24" height="24">
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="24" height="24">
|
||||
|
||||
</a>
|
||||
</div>
|
||||
@@ -196,7 +196,7 @@
|
||||
<label class="md-nav__title md-nav__title--site" for="drawer">
|
||||
<span class="md-nav__button md-logo">
|
||||
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="48" height="48">
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="48" height="48">
|
||||
|
||||
</span>
|
||||
LBRY
|
||||
@@ -304,4 +304,4 @@
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
@@ -43,19 +43,13 @@
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "If provided only the balance for this account will be given",
|
||||
"description": "If provided only the balance for this account will be given. Otherwise default account.",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "address",
|
||||
"type": "str",
|
||||
"description": "If provided only the balance for this address will be given",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "include_unconfirmed",
|
||||
"type": "bool",
|
||||
"description": "Include unconfirmed",
|
||||
"name": "confirmations",
|
||||
"type": "int",
|
||||
"description": "Only include transactions with this many confirmed blocks.",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
@@ -83,18 +77,25 @@
|
||||
{
|
||||
"name": "account_decrypt",
|
||||
"description": "Decrypt an encrypted account, this will remove the wallet password",
|
||||
"arguments": [],
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id for the account to decrypt",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if wallet is decrypted, otherwise false"
|
||||
},
|
||||
{
|
||||
"name": "account_encrypt",
|
||||
"description": "Encrypt a wallet with a password, if the wallet is already encrypted this will update\nthe password",
|
||||
"description": "Encrypt an unencrypted account with a password",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "new_password",
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "password string to be used for encrypting wallet",
|
||||
"is_required": true
|
||||
"description": "id for the account to encrypt",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if wallet is decrypted, otherwise false"
|
||||
@@ -107,13 +108,13 @@
|
||||
"name": "to_account",
|
||||
"type": "str",
|
||||
"description": "send to this account",
|
||||
"is_required": true
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "from_account",
|
||||
"type": "str",
|
||||
"description": "spend from this account",
|
||||
"is_required": true
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "amount",
|
||||
@@ -155,13 +156,7 @@
|
||||
{
|
||||
"name": "confirmations",
|
||||
"type": "int",
|
||||
"description": "required confirmations (default: 6)",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "include_reserved",
|
||||
"type": "bool",
|
||||
"description": "include reserved UTXOs (default: false)",
|
||||
"description": "required confirmations (default: 0)",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
@@ -179,6 +174,19 @@
|
||||
],
|
||||
"returns": "(map) balance of account(s)"
|
||||
},
|
||||
{
|
||||
"name": "account_lock",
|
||||
"description": "Lock an unlocked account",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id for the account to lock",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if account is locked, otherwise false"
|
||||
},
|
||||
{
|
||||
"name": "account_max_address_gap",
|
||||
"description": "Finds ranges of consecutive addresses that are unused and returns the length\nof the longest such range: for change and receiving address chains. This is\nuseful to figure out ideal values to set for 'receiving_gap' and 'change_gap'\naccount settings.",
|
||||
@@ -205,6 +213,25 @@
|
||||
],
|
||||
"returns": "(map) details of removed account"
|
||||
},
|
||||
{
|
||||
"name": "account_send",
|
||||
"description": "Send the same number of credits to multiple addresses.",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "account to fund the transaction",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "broadcast",
|
||||
"type": "bool",
|
||||
"description": "actually broadcast the transaction, default: false.",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": ""
|
||||
},
|
||||
{
|
||||
"name": "account_set",
|
||||
"description": "Change various settings on an account.",
|
||||
@@ -259,10 +286,10 @@
|
||||
"description": "Unlock an encrypted account",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "password",
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "password for unlocking wallet",
|
||||
"is_required": true
|
||||
"description": "id for the account to unlock",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if account is unlocked, otherwise false"
|
||||
@@ -295,23 +322,22 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to use",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "List of wallet addresses"
|
||||
},
|
||||
{
|
||||
"name": "address_public_key",
|
||||
"description": "Get public key from wallet address",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "address",
|
||||
"type": "str",
|
||||
"description": "address for which to get the public key",
|
||||
"is_required": true
|
||||
}
|
||||
],
|
||||
"returns": "(list) list of public keys associated with address.\n Could contain more than one public key if multisig."
|
||||
},
|
||||
{
|
||||
"name": "address_unused",
|
||||
"description": "Return an address containing no balance, will create\na new address if there is none.",
|
||||
@@ -535,7 +561,26 @@
|
||||
{
|
||||
"name": "channel_list",
|
||||
"description": "Get certificate claim infos for channels that can be published to",
|
||||
"arguments": [],
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id of the account to use",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet."
|
||||
},
|
||||
{
|
||||
@@ -553,6 +598,12 @@
|
||||
"type": "decimal",
|
||||
"description": "bid amount on the channel",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id of the account to store channel",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }"
|
||||
@@ -584,6 +635,12 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to use",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "blocking",
|
||||
"type": "bool",
|
||||
"description": "wait until abandon is in mempool",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(dict) Dictionary containing result of the claim\n {\n success: (bool) True if txn is successful\n txid : (str) txid of resulting transaction\n }"
|
||||
@@ -641,6 +698,18 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to query",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) \"claim\", \"update\" , or \"support\"\n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the claim\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]"
|
||||
@@ -676,25 +745,6 @@
|
||||
],
|
||||
"returns": "(dict) Dictionary containing the transaction information\n {\n \"hex\": (str) raw transaction,\n \"inputs\": (list) inputs(dict) used for the transaction,\n \"outputs\": (list) outputs(dict) for the transaction,\n \"total_fee\": (int) fee in dewies,\n \"total_input\": (int) total of inputs in dewies,\n \"total_output\": (int) total of outputs in dewies(input - fees),\n \"txid\": (str) txid of the transaction,\n }"
|
||||
},
|
||||
{
|
||||
"name": "claim_renew",
|
||||
"description": "Renew claim(s) or support(s)",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "outpoint",
|
||||
"type": "str",
|
||||
"description": "outpoint of the claim to renew",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "height",
|
||||
"type": "str",
|
||||
"description": "update claims expiring before or at this block height",
|
||||
"is_required": true
|
||||
}
|
||||
],
|
||||
"returns": "(dict) Dictionary where key is the the original claim's outpoint and\n value is the result of the renewal\n {\n outpoint:{\n\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n },\n }"
|
||||
},
|
||||
{
|
||||
"name": "claim_send_to_address",
|
||||
"description": "Send a name claim to an address",
|
||||
@@ -1190,10 +1240,22 @@
|
||||
"description": "claim id of the publisher channel, does not check for channel claim being in the wallet. This allows publishing to a channel where only the certificate private key is in the wallet.",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "channel_account_id",
|
||||
"type": "str",
|
||||
"description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "account to use for funding the transaction",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "claim_address",
|
||||
"type": "str",
|
||||
"description": "address where the claim is sent to, if not specified new address wil automatically be created",
|
||||
"description": "address where the claim is sent to, if not specified new address will automatically be created",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
@@ -1363,7 +1425,7 @@
|
||||
"name": "status",
|
||||
"description": "Get daemon status",
|
||||
"arguments": [],
|
||||
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }"
|
||||
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n },\n 'upnp': {\n 'aioupnp_version': (str),\n 'redirects': {\n <TCP | UDP>: (int) external_port,\n },\n 'gateway': (str) manufacturer and model,\n 'dht_redirect_set': (bool),\n 'peer_redirect_set': (bool),\n 'external_ip': (str) external ip address,\n }\n }"
|
||||
},
|
||||
{
|
||||
"name": "stop",
|
||||
@@ -1424,6 +1486,18 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to query",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) List of transactions\n\n {\n \"claim_info\": (list) claim info if in txn [{\n \"address\": (str) address of claim,\n \"balance_delta\": (float) bid amount,\n \"amount\": (float) claim amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"abandon_info\": (list) abandon info if in txn [{\n \"address\": (str) address of abandoned claim,\n \"balance_delta\": (float) returned amount,\n \"amount\": (float) claim amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"confirmations\": (int) number of confirmations for the txn,\n \"date\": (str) date and time of txn,\n \"fee\": (float) txn fee,\n \"support_info\": (list) support info if in txn [{\n \"address\": (str) address of support,\n \"balance_delta\": (float) support amount,\n \"amount\": (float) support amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"is_tip\": (bool),\n \"nout\": (int) nout\n }],\n \"timestamp\": (int) timestamp,\n \"txid\": (str) txn id,\n \"update_info\": (list) update info if in txn [{\n \"address\": (str) address of claim,\n \"balance_delta\": (float) credited/debited\n \"amount\": (float) absolute amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"value\": (float) value of txn\n }"
|
||||
@@ -1450,6 +1524,18 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to query",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) List of unspent transaction outputs (UTXOs)\n [\n {\n \"address\": (str) the output address\n \"amount\": (float) unspent amount\n \"height\": (int) block height\n \"is_claim\": (bool) is the tx a claim\n \"is_coinbase\": (bool) is the tx a coinbase tx\n \"is_support\": (bool) is the tx a support\n \"is_update\": (bool) is the tx an update\n \"nout\": (int) nout of the output\n \"txid\": (str) txid of the output\n },\n ...\n ]"
|
||||
|
||||
@@ -1903,7 +1903,7 @@ Options:
|
||||
publishing to a channel where only the certificate
|
||||
private key is in the wallet.
|
||||
--claim_address=<claim_address> : (str) address where the claim is sent to, if not specified
|
||||
new address wil automatically be created
|
||||
new address will automatically be created
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
|
||||
@@ -1743,7 +1743,7 @@ Args:
|
||||
publishing to a channel where only the certificate
|
||||
private key is in the wallet.
|
||||
'claim_address' : (str) address where the claim is sent to, if not specified
|
||||
new address wil automatically be created
|
||||
new address will automatically be created
|
||||
|
||||
Returns:
|
||||
(dict) Dictionary containing result of the claim
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"docs": [
|
||||
{
|
||||
"location": "/",
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
|
||||
"title": "API"
|
||||
},
|
||||
{
|
||||
@@ -167,7 +167,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/#publish",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"title": "publish"
|
||||
},
|
||||
{
|
||||
@@ -292,7 +292,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/",
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
|
||||
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
|
||||
"title": "CLI"
|
||||
},
|
||||
{
|
||||
@@ -457,7 +457,7 @@
|
||||
},
|
||||
{
|
||||
"location": "/cli/#publish",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
|
||||
"title": "publish"
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# This is an example daemon_settings.yml file.
|
||||
# See https://lbry.tech/resources/daemon-settings for all configuration keys and values
|
||||
|
||||
share_usage_data: True
|
||||
|
||||
lbryum_servers:
|
||||
- lbryumx1.lbry.io:50001
|
||||
- lbryumx2.lbry.io:50001
|
||||
|
||||
blockchain_name: lbrycrd_main
|
||||
|
||||
data_dir: /home/lbry/.lbrynet
|
||||
download_directory: /home/lbry/downloads
|
||||
|
||||
delete_blobs_on_remove: True
|
||||
dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
# - dht
|
||||
|
After Width: | Height: | Size: 7.4 KiB |
|
After Width: | Height: | Size: 19 KiB |
|
After Width: | Height: | Size: 1.2 KiB |
|
After Width: | Height: | Size: 2.6 KiB |
|
After Width: | Height: | Size: 6.1 KiB |
|
After Width: | Height: | Size: 97 KiB |
|
After Width: | Height: | Size: 1.1 KiB |
|
After Width: | Height: | Size: 361 KiB |
|
After Width: | Height: | Size: 5.3 KiB |
|
After Width: | Height: | Size: 15 KiB |
|
After Width: | Height: | Size: 31 KiB |
@@ -1,6 +1,4 @@
|
||||
import logging
|
||||
|
||||
__version__ = "0.30.0rc2"
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.30.5"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -1,293 +0,0 @@
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer, task
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import looping_call_manager, utils, system_info
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
SERVER_STARTUP_ERROR = 'Server Startup Error'
|
||||
DOWNLOAD_STARTED = 'Download Started'
|
||||
DOWNLOAD_ERRORED = 'Download Errored'
|
||||
DOWNLOAD_FINISHED = 'Download Finished'
|
||||
HEARTBEAT = 'Heartbeat'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
NEW_DOWNLOAD_STAT = 'Download'
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager:
|
||||
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
|
||||
self.analytics_api = analytics_api
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.looping_call_manager = self._setup_looping_calls()
|
||||
self.context = context or self._make_context(
|
||||
system_info.get_platform(), conf.settings['wallet'])
|
||||
self.installation_id = installation_id or conf.settings.installation_id
|
||||
self.session_id = session_id or conf.settings.get_session_id()
|
||||
self.is_started = False
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
api = Api.new_instance(enabled)
|
||||
return cls(api)
|
||||
|
||||
# Things We Track
|
||||
def send_new_download_start(self, download_id, name, claim_dict):
|
||||
self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
|
||||
def send_new_download_success(self, download_id, name, claim_dict):
|
||||
self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
|
||||
def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
|
||||
'message': str(e),
|
||||
})
|
||||
|
||||
def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
self.analytics_api.track({
|
||||
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
|
||||
'event': NEW_DOWNLOAD_STAT,
|
||||
'properties': self._event_properties({
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
|
||||
'action': action,
|
||||
'error': e,
|
||||
}),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow(),
|
||||
})
|
||||
|
||||
def send_server_startup(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP))
|
||||
|
||||
def send_server_startup_success(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
|
||||
def send_server_startup_error(self, message):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
def send_download_started(self, id_, name, claim_dict=None):
|
||||
self.analytics_api.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
def send_claim_action(self, action):
|
||||
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
def send_new_channel(self):
|
||||
self.analytics_api.track(self._event(NEW_CHANNEL))
|
||||
|
||||
def send_credits_sent(self):
|
||||
self.analytics_api.track(self._event(CREDITS_SENT))
|
||||
|
||||
def _send_heartbeat(self):
|
||||
self.analytics_api.track(self._event(HEARTBEAT))
|
||||
|
||||
def _update_tracked_metrics(self):
|
||||
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
# Setup / Shutdown
|
||||
|
||||
def start(self):
|
||||
if not self.is_started:
|
||||
for name, _, interval in self._get_looping_calls():
|
||||
self.looping_call_manager.start(name, interval)
|
||||
self.is_started = True
|
||||
|
||||
def shutdown(self):
|
||||
self.looping_call_manager.shutdown()
|
||||
|
||||
def register_repeating_metric(self, event_name, value_generator, frequency=300):
|
||||
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
|
||||
self.looping_call_manager.register_looping_call(event_name, lcall)
|
||||
lcall.start(frequency)
|
||||
|
||||
def _get_looping_calls(self):
|
||||
return [
|
||||
('send_heartbeat', self._send_heartbeat, 60),
|
||||
('update_tracked_metrics', self._update_tracked_metrics, 300),
|
||||
]
|
||||
|
||||
def _setup_looping_calls(self):
|
||||
call_manager = looping_call_manager.LoopingCallManager()
|
||||
for name, fn, _ in self._get_looping_calls():
|
||||
call_manager.register_looping_call(name, task.LoopingCall(fn))
|
||||
return call_manager
|
||||
|
||||
def _send_repeating_metric(self, event_name, value_generator):
|
||||
result = value_generator()
|
||||
self._if_deferred(result, self._send_repeating_metric_value, event_name)
|
||||
|
||||
def _send_repeating_metric_value(self, result, event_name):
|
||||
should_send, value = result
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(event_name, value))
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self._tracked_data[metric].append(value)
|
||||
|
||||
def summarize_and_reset(self, metric, op=sum):
|
||||
"""Apply `op` on the current values for `metric`.
|
||||
|
||||
This operation also resets the metric.
|
||||
|
||||
Returns:
|
||||
a tuple (should_send, value)
|
||||
"""
|
||||
try:
|
||||
values = self._tracked_data.pop(metric)
|
||||
return True, op(values)
|
||||
except KeyError:
|
||||
return False, None
|
||||
|
||||
def _event(self, event, event_properties=None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': self._event_properties(event_properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _metric_event(self, metric_name, value):
|
||||
return self._event(metric_name, {'value': value})
|
||||
|
||||
def _event_properties(self, event_properties=None):
|
||||
properties = {
|
||||
'lbry_id': self.installation_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
@staticmethod
|
||||
def _download_properties(id_, name, claim_dict=None, report=None):
|
||||
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
|
||||
p = {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
if report:
|
||||
p['report'] = report
|
||||
return p
|
||||
|
||||
@staticmethod
|
||||
def _download_error_properties(error, id_, name, claim_dict, report):
|
||||
def error_name(err):
|
||||
if not hasattr(type(err), "__name__"):
|
||||
return str(type(err))
|
||||
return type(err).__name__
|
||||
return {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': claim_dict.source_hash.decode(),
|
||||
'error': error_name(error),
|
||||
'reason': str(error),
|
||||
'report': report
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _make_context(platform, wallet):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
|
||||
if isinstance(maybe_deferred, defer.Deferred):
|
||||
maybe_deferred.addCallback(callback, *args, **kwargs)
|
||||
else:
|
||||
callback(maybe_deferred, *args, **kwargs)
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self, cookies, url, write_key, enabled):
|
||||
self.cookies = cookies
|
||||
self.url = url
|
||||
self._write_key = write_key
|
||||
self._enabled = enabled
|
||||
|
||||
def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
def update_cookies(response):
|
||||
self.cookies.update(response.cookies())
|
||||
return response
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
headers = {b"Connection": b"close"}
|
||||
d = treq.post(self.url + endpoint, auth=(self._write_key, ''), json=data,
|
||||
headers=headers, cookies=self.cookies)
|
||||
d.addCallback(update_cookies)
|
||||
return d
|
||||
|
||||
def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if not self._enabled:
|
||||
return defer.succeed('Analytics disabled')
|
||||
|
||||
def _log_error(failure, event):
|
||||
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
|
||||
|
||||
log.debug('Sending track event: %s', event)
|
||||
d = self._post('/track', event)
|
||||
d.addErrback(_log_error, event)
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
"""Initialize an instance using values from the configuration"""
|
||||
if enabled is None:
|
||||
enabled = conf.settings['share_usage_data']
|
||||
return cls(
|
||||
{},
|
||||
conf.settings['ANALYTICS_ENDPOINT'],
|
||||
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
|
||||
enabled,
|
||||
)
|
||||
@@ -1 +0,0 @@
|
||||
from . import paths
|
||||
@@ -1,26 +0,0 @@
|
||||
# App root files dir
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29
|
||||
def android_files_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base internal storage path
|
||||
def android_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base external (SD card, if present) storage path
|
||||
def android_external_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Internal device storage (private app folder)
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# External (app folder on SD card, if present) storage
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_external_storage_dir():
|
||||
return None
|
||||
@@ -7,7 +7,7 @@ from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from lbrynet.core.BlobInfo import BlobInfo
|
||||
from lbrynet.p2p.BlobInfo import BlobInfo
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -53,10 +53,10 @@ class StreamBlobDecryptor:
|
||||
|
||||
def decrypt(self, write_func):
|
||||
"""
|
||||
Decrypt blob and write its content useing write_func
|
||||
Decrypt blob and write its content using write_func
|
||||
|
||||
write_func - function that takes decrypted string as
|
||||
arugment and writes it somewhere
|
||||
argument and writes it somewhere
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -75,7 +75,7 @@ class StreamBlobDecryptor:
|
||||
def finish_decrypt():
|
||||
bytes_left = len(self.buff) % (AES.block_size // 8)
|
||||
if bytes_left != 0:
|
||||
log.warning(self.buff[-1 * (AES.block_size // 8):].encode('hex'))
|
||||
log.warning(binascii.hexlify(self.buff[-1 * (AES.block_size // 8):]).decode())
|
||||
raise Exception("blob %s has incorrect padding: %i bytes left" %
|
||||
(self.blob.blob_hash, bytes_left))
|
||||
data_to_decrypt, self.buff = self.buff, b''
|
||||
@@ -6,7 +6,7 @@ import logging
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from twisted.internet import defer
|
||||
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
|
||||
from lbrynet.blob.CryptBlob import CryptStreamBlobMaker
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -2,16 +2,17 @@
|
||||
Utilities for turning plain files into LBRY Files.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
|
||||
from lbrynet.core.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
|
||||
from lbrynet.core.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
|
||||
from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
|
||||
from lbrynet.blob.CryptStreamCreator import CryptStreamCreator
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -119,13 +120,13 @@ def create_lbry_file(blob_manager, storage, payment_rate_manager, lbry_file_mana
|
||||
sd_hash = yield descriptor_writer.create_descriptor(sd_info)
|
||||
|
||||
log.debug("saving the stream")
|
||||
yield storage.store_stream(
|
||||
yield f2d(storage.store_stream(
|
||||
sd_info['stream_hash'], sd_hash, sd_info['stream_name'], sd_info['key'],
|
||||
sd_info['suggested_file_name'], sd_info['blobs']
|
||||
)
|
||||
))
|
||||
log.debug("adding to the file manager")
|
||||
lbry_file = yield lbry_file_manager.add_published_file(
|
||||
lbry_file = yield f2d(lbry_file_manager.add_published_file(
|
||||
sd_info['stream_hash'], sd_hash, hexlify(file_directory.encode()), payment_rate_manager,
|
||||
payment_rate_manager.min_blob_data_payment_rate
|
||||
)
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
@@ -5,14 +5,14 @@ import logging
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.core.HTTPBlobDownloader import HTTPBlobDownloader
|
||||
from lbrynet.core.utils import short_hash
|
||||
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaver
|
||||
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileDownloader
|
||||
from lbrynet.file_manager.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.core.StreamDescriptor import save_sd_info
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
|
||||
from lbrynet.utils import short_hash
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
|
||||
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -34,12 +34,12 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage, lbry_file_manager,
|
||||
payment_rate_manager, wallet, download_directory, file_name, stream_name, sd_hash, key,
|
||||
suggested_file_name, download_mirrors=None):
|
||||
def __init__(self, conf: Config, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
|
||||
lbry_file_manager, payment_rate_manager, wallet, download_directory, file_name, stream_name,
|
||||
sd_hash, key, suggested_file_name, download_mirrors=None):
|
||||
super().__init__(
|
||||
stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name
|
||||
conf, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager,
|
||||
wallet, download_directory, key, stream_name, file_name
|
||||
)
|
||||
self.sd_hash = sd_hash
|
||||
self.rowid = rowid
|
||||
@@ -55,9 +55,9 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.channel_name = None
|
||||
self.metadata = None
|
||||
self.mirror = None
|
||||
if download_mirrors or conf.settings['download_mirrors']:
|
||||
if download_mirrors or conf.download_mirrors:
|
||||
self.mirror = HTTPBlobDownloader(
|
||||
self.blob_manager, servers=download_mirrors or conf.settings['download_mirrors']
|
||||
self.blob_manager, servers=download_mirrors or conf.download_mirrors
|
||||
)
|
||||
|
||||
def set_claim_info(self, claim_info):
|
||||
@@ -70,13 +70,11 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.channel_name = claim_info['channel_name']
|
||||
self.metadata = claim_info['value']['stream']['metadata']
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claim_info(self, include_supports=True):
|
||||
claim_info = yield self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
async def get_claim_info(self, include_supports=True):
|
||||
claim_info = await self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
if claim_info:
|
||||
self.set_claim_info(claim_info)
|
||||
|
||||
defer.returnValue(claim_info)
|
||||
return claim_info
|
||||
|
||||
@property
|
||||
def saving_status(self):
|
||||
@@ -93,7 +91,7 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
|
||||
self.completed = True
|
||||
else:
|
||||
raise Exception("Unknown status for stream %s: %s" % (self.stream_hash, status))
|
||||
raise Exception(f"Unknown status for stream {self.stream_hash}: {status}")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self, err=None, change_status=True):
|
||||
@@ -101,16 +99,15 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
if self.mirror:
|
||||
self.mirror.stop()
|
||||
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
yield EncryptedFileDownloader.stop(self, err=err)
|
||||
yield super().stop(err)
|
||||
if change_status is True:
|
||||
status = yield self._save_status()
|
||||
defer.returnValue(status)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def status(self):
|
||||
blobs = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
async def status(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
|
||||
completed_blobs = yield self.blob_manager.completed_blobs(blob_hashes)
|
||||
completed_blobs = self.blob_manager.completed_blobs(blob_hashes)
|
||||
num_blobs_completed = len(completed_blobs)
|
||||
num_blobs_known = len(blob_hashes)
|
||||
|
||||
@@ -120,9 +117,10 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
status = "stopped"
|
||||
else:
|
||||
status = "running"
|
||||
defer.returnValue(EncryptedFileStatusReport(
|
||||
|
||||
return EncryptedFileStatusReport(
|
||||
self.file_name, num_blobs_completed, num_blobs_known, status
|
||||
))
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start(self):
|
||||
@@ -150,7 +148,7 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_RUNNING
|
||||
status = yield self.lbry_file_manager.change_lbry_file_status(self, status)
|
||||
self._saving_status = False
|
||||
defer.returnValue(status)
|
||||
return status
|
||||
|
||||
def save_status(self):
|
||||
return self._save_status()
|
||||
@@ -180,10 +178,10 @@ class ManagedEncryptedFileDownloaderFactory:
|
||||
if file_name:
|
||||
file_name = hexlify(file_name.encode())
|
||||
hex_download_directory = hexlify(download_directory.encode())
|
||||
lbry_file = yield self.lbry_file_manager.add_downloaded_file(
|
||||
lbry_file = yield f2d(self.lbry_file_manager.add_downloaded_file(
|
||||
stream_hash, metadata.source_blob_hash, hex_download_directory, payment_rate_manager,
|
||||
data_rate, file_name=file_name, download_mirrors=download_mirrors
|
||||
)
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
@staticmethod
|
||||
@@ -3,19 +3,20 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
import random
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.reflector.reupload import reflect_file
|
||||
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from lbrynet.core.StreamDescriptor import EncryptedFileStreamType, get_sd_info
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import AlreadyStoppedError
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CurrentlyStoppingError
|
||||
from lbrynet.core.utils import safe_start_looping_call, safe_stop_looping_call
|
||||
from lbrynet import conf
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.reflector.reupload import reflect_file
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, get_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import AlreadyStoppedError
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CurrentlyStoppingError
|
||||
from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,9 +29,11 @@ class EncryptedFileManager:
|
||||
# when reflecting files, reflect up to this many files at a time
|
||||
CONCURRENT_REFLECTS = 5
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, wallet, payment_rate_manager, storage, sd_identifier):
|
||||
self.auto_re_reflect = conf.settings['reflect_uploads'] and conf.settings['auto_re_reflect_interval'] > 0
|
||||
self.auto_re_reflect_interval = conf.settings['auto_re_reflect_interval']
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, wallet,
|
||||
payment_rate_manager, storage, sd_identifier):
|
||||
self.conf = conf
|
||||
self.auto_re_reflect = conf.reflect_uploads and conf.auto_re_reflect_interval > 0
|
||||
self.auto_re_reflect_interval = conf.auto_re_reflect_interval
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
@@ -42,11 +45,9 @@ class EncryptedFileManager:
|
||||
self.lbry_files = []
|
||||
self.lbry_file_reflector = task.LoopingCall(self.reflect_lbry_files)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
yield self._add_to_sd_identifier()
|
||||
yield self._start_lbry_files()
|
||||
log.info("Started file manager")
|
||||
self._add_to_sd_identifier()
|
||||
return self._start_lbry_files()
|
||||
|
||||
def get_lbry_file_status(self, lbry_file):
|
||||
return self.storage.get_lbry_file_status(lbry_file.rowid)
|
||||
@@ -56,7 +57,7 @@ class EncryptedFileManager:
|
||||
|
||||
def change_lbry_file_status(self, lbry_file, status):
|
||||
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
|
||||
return self.storage.change_file_status(lbry_file.rowid, status)
|
||||
return f2d(self.storage.change_file_status(lbry_file.rowid, status))
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
@@ -80,6 +81,7 @@ class EncryptedFileManager:
|
||||
def _get_lbry_file(self, rowid, stream_hash, payment_rate_manager, sd_hash, key,
|
||||
stream_name, file_name, download_directory, suggested_file_name, download_mirrors=None):
|
||||
return ManagedEncryptedFileDownloader(
|
||||
self.conf,
|
||||
rowid,
|
||||
stream_hash,
|
||||
self.peer_finder,
|
||||
@@ -116,10 +118,9 @@ class EncryptedFileManager:
|
||||
except Exception:
|
||||
log.warning("Failed to start %i", file_info.get('rowid'))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start_lbry_files(self):
|
||||
files = yield self.storage.get_all_lbry_files()
|
||||
claim_infos = yield self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
async def _start_lbry_files(self):
|
||||
files = await self.storage.get_all_lbry_files()
|
||||
claim_infos = await self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
prm = self.payment_rate_manager
|
||||
|
||||
log.info("Starting %i files", len(files))
|
||||
@@ -152,14 +153,13 @@ class EncryptedFileManager:
|
||||
log.info("Stopping %i lbry files", len(self.lbry_files))
|
||||
yield defer.DeferredList([self._stop_lbry_file(lbry_file) for lbry_file in list(self.lbry_files)])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
async def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = stream_metadata['suggested_file_name']
|
||||
rowid = yield self.storage.save_published_file(
|
||||
rowid = await self.storage.save_published_file(
|
||||
stream_hash, file_name, download_directory, blob_data_rate, status
|
||||
)
|
||||
lbry_file = self._get_lbry_file(
|
||||
@@ -167,37 +167,36 @@ class EncryptedFileManager:
|
||||
stream_metadata['suggested_file_name'], download_mirrors=None
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info()
|
||||
await lbry_file.get_claim_info()
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
async def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
blob_data_rate=None, status=None, file_name=None, download_mirrors=None):
|
||||
status = status or ManagedEncryptedFileDownloader.STATUS_STOPPED
|
||||
payment_rate_manager = payment_rate_manager or self.payment_rate_manager
|
||||
blob_data_rate = blob_data_rate or payment_rate_manager.min_blob_data_payment_rate
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = file_name or stream_metadata['suggested_file_name']
|
||||
|
||||
# when we save the file we'll atomic touch the nearest file to the suggested file name
|
||||
# that doesn't yet exist in the download directory
|
||||
rowid = yield self.storage.save_downloaded_file(
|
||||
rowid = await self.storage.save_downloaded_file(
|
||||
stream_hash, hexlify(os.path.basename(unhexlify(file_name))), download_directory, blob_data_rate
|
||||
)
|
||||
file_name = (yield self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
file_name = (await self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
lbry_file = self._get_lbry_file(
|
||||
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
|
||||
stream_metadata['suggested_file_name'], download_mirrors
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info(include_supports=False)
|
||||
await lbry_file.get_claim_info(include_supports=False)
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_lbry_file(self, lbry_file, delete_file=False):
|
||||
@@ -223,7 +222,7 @@ class EncryptedFileManager:
|
||||
del self.storage.content_claim_callbacks[lbry_file.stream_hash]
|
||||
|
||||
yield lbry_file.delete_data()
|
||||
yield self.storage.delete_stream(lbry_file.stream_hash)
|
||||
yield f2d(self.storage.delete_stream(lbry_file.stream_hash))
|
||||
|
||||
if delete_file and os.path.isfile(full_path):
|
||||
os.remove(full_path)
|
||||
@@ -241,10 +240,10 @@ class EncryptedFileManager:
|
||||
def reflect_lbry_files(self):
|
||||
sem = defer.DeferredSemaphore(self.CONCURRENT_REFLECTS)
|
||||
ds = []
|
||||
sd_hashes_to_reflect = yield self.storage.get_streams_to_re_reflect()
|
||||
sd_hashes_to_reflect = yield f2d(self.storage.get_streams_to_re_reflect())
|
||||
for lbry_file in self.lbry_files:
|
||||
if lbry_file.sd_hash in sd_hashes_to_reflect:
|
||||
ds.append(sem.run(reflect_file, lbry_file))
|
||||
ds.append(sem.run(reflect_file, lbry_file, random.choice(self.conf.reflector_servers)))
|
||||
yield defer.DeferredList(ds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -1,4 +0,0 @@
|
||||
from .blob_file import BlobFile
|
||||
from .creator import BlobFileCreator
|
||||
from .writer import HashBlobWriter
|
||||
from .reader import HashBlobReader
|
||||
@@ -1,10 +1,10 @@
|
||||
import logging
|
||||
import os
|
||||
from twisted.internet import defer, threads
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
from lbrynet.blob.reader import HashBlobReader
|
||||
|
||||
@@ -12,6 +12,24 @@ log = logging.getLogger(__name__)
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char):
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash):
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@param blobhash: string, the blobhash to check
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
class BlobFile:
|
||||
"""
|
||||
@@ -52,7 +70,7 @@ class BlobFile:
|
||||
def open_for_writing(self, peer):
|
||||
"""
|
||||
open a blob file to be written by peer, supports concurrent
|
||||
writers, as long as they are from differnt peers.
|
||||
writers, as long as they are from different peers.
|
||||
|
||||
returns tuple of (writer, finished_deferred)
|
||||
|
||||
@@ -93,29 +111,19 @@ class BlobFile:
|
||||
if not self.writers and not self.readers:
|
||||
self._verified = False
|
||||
self.saved_verified_blob = False
|
||||
|
||||
def delete_from_file_system():
|
||||
try:
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
log.warning("An error occurred deleting %s: %s",
|
||||
str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
except Exception as e:
|
||||
log.exception("An error occurred deleting %s:", str(self.file_path), exc_info=e)
|
||||
else:
|
||||
return defer.fail(Failure(
|
||||
ValueError("File is currently being read or written and cannot be deleted")))
|
||||
raise ValueError("File is currently being read or written and cannot be deleted")
|
||||
|
||||
@property
|
||||
def verified(self):
|
||||
"""
|
||||
Protect verified from being modified by other classes.
|
||||
verified is True if a write to a blob has completed succesfully,
|
||||
verified is True if a write to a blob has completed successfully,
|
||||
or a blob has been read to have the same length as specified
|
||||
in init
|
||||
"""
|
||||
@@ -182,7 +190,7 @@ class BlobFile:
|
||||
if writer.len_so_far != self.length:
|
||||
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
|
||||
else:
|
||||
err_string = "blob hash is %s vs expected %s" % (writer.blob_hash, self.blob_hash)
|
||||
err_string = f"blob hash is {writer.blob_hash} vs expected {self.blob_hash}"
|
||||
errback_finished_deferred(Failure(InvalidDataError(err_string)))
|
||||
d = defer.succeed(None)
|
||||
else:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import binascii
|
||||
from twisted.internet import defer
|
||||
from lbrynet.cryptstream.CryptBlob import StreamBlobDecryptor
|
||||
from lbrynet.blob.CryptBlob import StreamBlobDecryptor
|
||||
|
||||
|
||||
class CryptBlobHandler:
|
||||
@@ -1,13 +1,15 @@
|
||||
from binascii import unhexlify
|
||||
import logging
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.core.client.DownloadManager import DownloadManager
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.cryptstream.client.CryptBlobHandler import CryptBlobHandler
|
||||
from binascii import unhexlify
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.p2p.client.BlobRequester import BlobRequester
|
||||
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.p2p.client.DownloadManager import DownloadManager
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -36,7 +38,7 @@ class CryptStreamDownloader:
|
||||
|
||||
#implements(IStreamDownloader)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
|
||||
key, stream_name):
|
||||
"""Initialize a CryptStreamDownloader
|
||||
|
||||
@@ -54,7 +56,7 @@ class CryptStreamDownloader:
|
||||
@return:
|
||||
|
||||
"""
|
||||
|
||||
self.conf = conf
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
@@ -2,14 +2,16 @@ import os
|
||||
import logging
|
||||
import traceback
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from lbrynet.core.StreamDescriptor import save_sd_info
|
||||
from lbrynet.cryptstream.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.core.Error import FileOpenError
|
||||
from lbrynet.lbry_file.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
|
||||
from twisted.internet import defer, threads
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.Error import FileOpenError
|
||||
from lbrynet.blob.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -17,9 +19,9 @@ log = logging.getLogger(__name__)
|
||||
class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
"""Classes which inherit from this class download LBRY files"""
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
storage, payment_rate_manager, wallet, key, stream_name, file_name):
|
||||
super().__init__(peer_finder, rate_limiter, blob_manager,
|
||||
super().__init__(conf, peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, key, stream_name)
|
||||
self.stream_hash = stream_hash
|
||||
self.storage = storage
|
||||
@@ -28,24 +30,34 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_data(self):
|
||||
crypt_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
crypt_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
blob_hashes = [b.blob_hash for b in crypt_infos if b.blob_hash]
|
||||
sd_hash = yield self.storage.get_sd_blob_hash_for_stream(self.stream_hash)
|
||||
sd_hash = yield f2d(self.storage.get_sd_blob_hash_for_stream(self.stream_hash))
|
||||
blob_hashes.append(sd_hash)
|
||||
yield self.blob_manager.delete_blobs(blob_hashes)
|
||||
|
||||
def stop(self, err=None):
|
||||
d = self._close_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
|
||||
return d
|
||||
self._close_output()
|
||||
return super().stop(err=err)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
def _start(self):
|
||||
def check_start_succeeded(success):
|
||||
if success:
|
||||
self.starting = False
|
||||
self.stopped = False
|
||||
self.completed = False
|
||||
return True
|
||||
else:
|
||||
return self._start_failed()
|
||||
|
||||
self.download_manager = self._get_download_manager()
|
||||
d = self._setup_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader._start(self))
|
||||
d.addCallback(lambda _: self.download_manager.start_downloading())
|
||||
d.addCallbacks(check_start_succeeded)
|
||||
return d
|
||||
|
||||
def _setup_output(self):
|
||||
@@ -54,14 +66,9 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
def _close_output(self):
|
||||
pass
|
||||
|
||||
def get_total_bytes(self):
|
||||
d = self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def calculate_size(blobs):
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
d.addCallback(calculate_size)
|
||||
return d
|
||||
async def get_total_bytes(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
def get_total_bytes_cached(self):
|
||||
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
|
||||
@@ -91,7 +98,8 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
class EncryptedFileDownloaderFactory:
|
||||
#implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet):
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet):
|
||||
self.conf = conf
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
@@ -124,9 +132,9 @@ class EncryptedFileDownloaderFactory:
|
||||
|
||||
|
||||
class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name):
|
||||
super().__init__(stream_hash, peer_finder, rate_limiter,
|
||||
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
|
||||
payment_rate_manager, wallet, download_directory, key, stream_name, file_name):
|
||||
super().__init__(conf, stream_hash, peer_finder, rate_limiter,
|
||||
blob_manager, storage, payment_rate_manager,
|
||||
wallet, key, stream_name, file_name)
|
||||
self.download_directory = unhexlify(download_directory).decode()
|
||||
@@ -136,10 +144,6 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __str__(self):
|
||||
return str(self.file_written_to)
|
||||
|
||||
def stop(self, err=None):
|
||||
d = EncryptedFileDownloader.stop(self, err=err)
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager,
|
||||
@@ -162,15 +166,11 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
|
||||
def _close_output(self):
|
||||
self.file_handle, file_handle = None, self.file_handle
|
||||
|
||||
def close_file():
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
return threads.deferToThread(close_file)
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
def _get_write_func(self):
|
||||
def write_func(data):
|
||||
@@ -180,8 +180,8 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
|
||||
|
||||
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
|
||||
super().__init__(peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
|
||||
super().__init__(conf, peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
self.download_directory = hexlify(download_directory.encode())
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
@@ -189,8 +189,9 @@ class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
key = stream_info.raw_info['key']
|
||||
suggested_file_name = stream_info.raw_info['suggested_file_name']
|
||||
return EncryptedFileSaver(
|
||||
stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage, payment_rate_manager,
|
||||
self.wallet, self.download_directory, key=key, stream_name=stream_name, file_name=suggested_file_name
|
||||
self.conf, stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage,
|
||||
payment_rate_manager, self.wallet, self.download_directory, key=key, stream_name=stream_name,
|
||||
file_name=suggested_file_name
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -17,9 +18,8 @@ class EncryptedFileMetadataHandler:
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_initial_blobs(self):
|
||||
blob_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
formatted_infos = self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
defer.returnValue(formatted_infos)
|
||||
blob_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
return self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
|
||||
def final_blob_num(self):
|
||||
return self._final_blob_num
|
||||
@@ -1,6 +1,6 @@
|
||||
from lbrynet.core.StreamDescriptor import EncryptedFileStreamType
|
||||
from lbrynet.core.StreamDescriptor import EncryptedFileStreamDescriptorValidator
|
||||
from lbrynet.core.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamDescriptorValidator
|
||||
from lbrynet.p2p.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
|
||||
|
||||
def add_lbry_file_to_sd_identifier(sd_identifier):
|
||||
@@ -3,7 +3,7 @@ import logging
|
||||
from io import BytesIO
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
# dont touch this. CI server changes this during build/deployment
|
||||
# don't touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
@@ -1,167 +0,0 @@
|
||||
import sys
|
||||
from twisted.internet import asyncioreactor
|
||||
if 'twisted.internet.reactor' not in sys.modules:
|
||||
asyncioreactor.install()
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
|
||||
# pyinstaller hooks install the default reactor before
|
||||
# any of our code runs, see kivy for similar problem:
|
||||
# https://github.com/kivy/kivy/issues/4182
|
||||
del sys.modules['twisted.internet.reactor']
|
||||
asyncioreactor.install()
|
||||
|
||||
import json
|
||||
import asyncio
|
||||
from aiohttp.client_exceptions import ClientConnectorError
|
||||
from requests.exceptions import ConnectionError
|
||||
from docopt import docopt
|
||||
from textwrap import dedent
|
||||
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
from lbrynet.daemon.DaemonControl import start as daemon_main
|
||||
from lbrynet.daemon.DaemonConsole import main as daemon_console
|
||||
from lbrynet.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.core.system_info import get_platform
|
||||
|
||||
|
||||
async def execute_command(method, params, conf_path=None):
|
||||
# this check if the daemon is running or not
|
||||
try:
|
||||
api = await LBRYAPIClient.get_client(conf_path)
|
||||
await api.status()
|
||||
except (ClientConnectorError, ConnectionError):
|
||||
await api.session.close()
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
return 1
|
||||
|
||||
# this actually executes the method
|
||||
try:
|
||||
resp = await api.call(method, params)
|
||||
await api.session.close()
|
||||
print(json.dumps(resp["result"], indent=2))
|
||||
except KeyError:
|
||||
if resp["error"]["code"] == -32500:
|
||||
print(json.dumps(resp["error"], indent=2))
|
||||
else:
|
||||
print(json.dumps(resp["error"]["message"], indent=2))
|
||||
|
||||
|
||||
def print_help():
|
||||
print(dedent("""
|
||||
NAME
|
||||
lbrynet - LBRY command line client.
|
||||
|
||||
USAGE
|
||||
lbrynet [--conf <config file>] <command> [<args>]
|
||||
|
||||
EXAMPLES
|
||||
lbrynet commands # list available commands
|
||||
lbrynet status # get daemon status
|
||||
lbrynet --conf ~/l1.conf status # like above but using ~/l1.conf as config file
|
||||
lbrynet resolve_name what # resolve a name
|
||||
lbrynet help resolve_name # get help for a command
|
||||
"""))
|
||||
|
||||
|
||||
def print_help_for_command(command):
|
||||
fn = Daemon.callable_methods.get(command)
|
||||
if fn:
|
||||
print(dedent(fn.__doc__))
|
||||
else:
|
||||
print("Invalid command name")
|
||||
|
||||
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
|
||||
return x
|
||||
if x.lower() == 'true':
|
||||
return True
|
||||
if x.lower() == 'false':
|
||||
return False
|
||||
if x.isdigit():
|
||||
return int(x)
|
||||
return x
|
||||
|
||||
|
||||
def remove_brackets(key):
|
||||
if key.startswith("<") and key.endswith(">"):
|
||||
return str(key[1:-1])
|
||||
return key
|
||||
|
||||
|
||||
def set_kwargs(parsed_args):
|
||||
kwargs = {}
|
||||
for key, arg in parsed_args.items():
|
||||
k = None
|
||||
if arg is None:
|
||||
continue
|
||||
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
kwargs[k] = normalize_value(arg, k)
|
||||
return kwargs
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
if not argv:
|
||||
print_help()
|
||||
return 1
|
||||
|
||||
conf_path = None
|
||||
if len(argv) and argv[0] == "--conf":
|
||||
if len(argv) < 2:
|
||||
print("No config file specified for --conf option")
|
||||
print_help()
|
||||
return 1
|
||||
|
||||
conf_path = argv[1]
|
||||
argv = argv[2:]
|
||||
|
||||
method, args = argv[0], argv[1:]
|
||||
|
||||
if method in ['help', '--help', '-h']:
|
||||
if len(args) == 1:
|
||||
print_help_for_command(args[0])
|
||||
else:
|
||||
print_help()
|
||||
return 0
|
||||
|
||||
elif method in ['version', '--version', '-v']:
|
||||
print(json.dumps(get_platform(get_ip=False), sort_keys=True, indent=2, separators=(',', ': ')))
|
||||
return 0
|
||||
|
||||
elif method == 'start':
|
||||
sys.exit(daemon_main(args, conf_path))
|
||||
|
||||
elif method == 'console':
|
||||
sys.exit(daemon_console())
|
||||
|
||||
elif method not in Daemon.callable_methods:
|
||||
if method not in Daemon.deprecated_methods:
|
||||
print('{} is not a valid command.'.format(method))
|
||||
return 1
|
||||
|
||||
new_method = Daemon.deprecated_methods[method].new_command
|
||||
if new_method is None:
|
||||
print("{} is permanently deprecated and does not have a replacement command.".format(method))
|
||||
return 0
|
||||
|
||||
print("{} is deprecated, using {}.".format(method, new_method))
|
||||
method = new_method
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
parsed = docopt(fn.__doc__, args)
|
||||
params = set_kwargs(parsed)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(execute_command(method, params, conf_path))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,59 +0,0 @@
|
||||
import platform
|
||||
import json
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
from six.moves.urllib import request
|
||||
from six.moves.urllib.error import URLError
|
||||
from lbryschema import __version__ as lbryschema_version
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
from lbrynet.conf import ROOT_DIR
|
||||
import logging.handlers
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_lbrynet_version():
|
||||
if build_type.BUILD == "dev":
|
||||
try:
|
||||
with open(os.devnull, 'w') as devnull:
|
||||
git_dir = ROOT_DIR + '/.git'
|
||||
return subprocess.check_output(
|
||||
['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'],
|
||||
stderr=devnull
|
||||
).decode().strip().lstrip('v')
|
||||
except (subprocess.CalledProcessError, OSError):
|
||||
log.debug("failed to get version from git")
|
||||
return lbrynet_version
|
||||
|
||||
|
||||
def get_platform(get_ip=True):
|
||||
p = {
|
||||
"processor": platform.processor(),
|
||||
"python_version": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"lbrynet_version": get_lbrynet_version(),
|
||||
"lbryschema_version": lbryschema_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
try:
|
||||
import distro
|
||||
p["distro"] = distro.info()
|
||||
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
|
||||
except ModuleNotFoundError:
|
||||
pass
|
||||
|
||||
# TODO: remove this from get_platform and add a get_external_ip function using treq
|
||||
if get_ip:
|
||||
try:
|
||||
response = json.loads(request.urlopen("https://api.lbry.io/ip").read())
|
||||
if not response['success']:
|
||||
raise URLError("failed to get external ip")
|
||||
p['ip'] = response['data']['ip']
|
||||
except (URLError, AssertionError):
|
||||
p['ip'] = "Could not determine IP"
|
||||
|
||||
return p
|
||||
@@ -1,8 +0,0 @@
|
||||
"""
|
||||
Classes and functions for dealing with Crypt Streams.
|
||||
|
||||
Crypt Streams are encrypted blobs and metadata tying those blobs together. At least some of the
|
||||
metadata is generally stored in a Stream Descriptor File, for example containing a public key
|
||||
used to bind blobs to the stream and a symmetric key used to encrypt the blobs. The list of blobs
|
||||
may or may not be present.
|
||||
"""
|
||||
@@ -1,106 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import inspect
|
||||
import logging
|
||||
TRACE = 5
|
||||
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
#
|
||||
# _srcfile is used when walking the stack to check when we've got the first
|
||||
# caller stack frame.
|
||||
#
|
||||
if hasattr(sys, 'frozen'): # support for py2exe
|
||||
_srcfile = "logging%s__init__%s" % (os.sep, __file__[-4:])
|
||||
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
|
||||
_srcfile = __file__[:-4] + '.py'
|
||||
else:
|
||||
_srcfile = __file__
|
||||
_srcfile = os.path.normcase(_srcfile)
|
||||
|
||||
|
||||
def findCaller(srcfile=None):
|
||||
"""Returns the filename, line number and function name of the caller"""
|
||||
srcfile = srcfile or _srcfile
|
||||
f = inspect.currentframe()
|
||||
# On some versions of IronPython, currentframe() returns None if
|
||||
# IronPython isn't run with -X:Frames.
|
||||
if f is not None:
|
||||
f = f.f_back
|
||||
rv = "(unknown file)", 0, "(unknown function)"
|
||||
while hasattr(f, "f_code"):
|
||||
co = f.f_code
|
||||
filename = os.path.normcase(co.co_filename)
|
||||
# ignore any function calls that are in this file
|
||||
if filename == srcfile:
|
||||
f = f.f_back
|
||||
continue
|
||||
rv = (filename, f.f_lineno, co.co_name)
|
||||
break
|
||||
return rv
|
||||
|
||||
|
||||
###
|
||||
|
||||
class Logger(logging.Logger):
|
||||
"""A logger that has an extra `fail` method useful for handling twisted failures."""
|
||||
|
||||
def fail(self, callback=None, *args, **kwargs):
|
||||
"""Returns a function to log a failure from an errback.
|
||||
|
||||
The returned function appends the error message and extracts
|
||||
the traceback from `err`.
|
||||
|
||||
Example usage:
|
||||
d.addErrback(log.fail(), 'This is an error message')
|
||||
|
||||
Although odd, making the method call is necessary to extract
|
||||
out useful filename and line number information; otherwise the
|
||||
reported values are from inside twisted's deferred handling
|
||||
code.
|
||||
|
||||
Args:
|
||||
callback: callable to call after making the log. The first argument
|
||||
will be the `err` from the deferred
|
||||
args: extra arguments to pass into `callback`
|
||||
|
||||
Returns: a function that takes the following arguments:
|
||||
err: twisted.python.failure.Failure
|
||||
msg: the message to log, using normal logging string iterpolation.
|
||||
msg_args: the values to subtitute into `msg`
|
||||
msg_kwargs: set `level` to change from the default ERROR severity. Other
|
||||
keywoards are treated as normal log kwargs.
|
||||
"""
|
||||
fn, lno, func = findCaller()
|
||||
|
||||
def _fail(err, msg, *msg_args, **msg_kwargs):
|
||||
level = msg_kwargs.pop('level', logging.ERROR)
|
||||
msg += ": %s"
|
||||
msg_args += (err.getErrorMessage(),)
|
||||
exc_info = (err.type, err.value, err.getTracebackObject())
|
||||
record = self.makeRecord(
|
||||
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
|
||||
self.handle(record)
|
||||
if callback:
|
||||
try:
|
||||
return callback(err, *args, **kwargs)
|
||||
except Exception:
|
||||
# log.fail is almost always called within an
|
||||
# errback. If callback fails and we didn't catch
|
||||
# the exception we would need to attach a second
|
||||
# errback to deal with that, which we will almost
|
||||
# never do and then we end up with an unhandled
|
||||
# error that will get swallowed by twisted
|
||||
self.exception('Failed to run callback')
|
||||
|
||||
return _fail
|
||||
|
||||
def trace(self, msg, *args, **kwargs):
|
||||
if self.isEnabledFor(TRACE):
|
||||
self._log(TRACE, msg, args, **kwargs)
|
||||
|
||||
|
||||
logging.setLoggerClass(Logger)
|
||||
logging.addLevelName(TRACE, 'TRACE')
|
||||
@@ -1,232 +0,0 @@
|
||||
import sys
|
||||
import code
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging.handlers
|
||||
from twisted.internet import defer, reactor, threads
|
||||
from aiohttp import client_exceptions
|
||||
|
||||
from lbrynet import analytics
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core import log_support
|
||||
from lbrynet.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
if sys.platform.startswith('darwin') or sys.platform.startswith('linux'):
|
||||
def color(msg, c="white"):
|
||||
_colors = {
|
||||
"normal": (0, 37),
|
||||
"underlined": (2, 37),
|
||||
"red": (1, 31),
|
||||
"green": (1, 32),
|
||||
"yellow": (1, 33),
|
||||
"blue": (1, 33),
|
||||
"magenta": (1, 34),
|
||||
"cyan": (1, 35),
|
||||
"white": (1, 36),
|
||||
"grey": (1, 37)
|
||||
}
|
||||
i, j = _colors[c]
|
||||
return "\033[%i;%i;40m%s\033[0m" % (i, j, msg)
|
||||
|
||||
|
||||
logo = """\
|
||||
╓▄█▄ç
|
||||
,▄█▓▓▀▀▀▓▓▓▌▄,
|
||||
▄▄▓▓▓▀¬ ╙▀█▓▓▓▄▄
|
||||
,▄█▓▓▀▀ ^▀▀▓▓▓▌▄,
|
||||
▄█▓▓█▀` ╙▀█▓▓▓▄▄
|
||||
╓▄▓▓▓▀╙ ▀▀▓▓▓▌▄,
|
||||
▄█▓▓█▀ ╙▀▓▓
|
||||
╓▄▓▓▓▀` ▄█▓▓▓▀
|
||||
▓▓█▀ ,▄▓▓▓▀╙
|
||||
▓▓m ╟▌▄, ▄█▓▓█▀ ,,╓µ
|
||||
▓▓m ^▀█▓▓▓▄▄ ╓▄▓▓▓▀╙ █▓▓▓▓▓▀
|
||||
▓▓Q '▀▀▓▓▓▌▄, ,▄█▓▓█▀ ▄█▓▓▓▓▓▀
|
||||
▀▓▓▓▌▄, ╙▀█▓▓█▄╗ ╓▄▓▓▓▀ ╓▄▓▓▓▀▀ ▀▀
|
||||
╙▀█▓▓█▄╗ ^▀▀▓▓▓▌▄▄█▓▓▀▀ ▄█▓▓█▀`
|
||||
'▀▀▓▓▓▌▄, ╙▀██▀` ╓▄▓▓▓▀╙
|
||||
╙▀█▓▓█▄╥ ,▄█▓▓▀▀
|
||||
└▀▀▓▓▓▌▄ ▄▒▓▓▓▀╙
|
||||
╙▀█▓▓█▓▓▓▀▀
|
||||
╙▀▀`
|
||||
|
||||
"""
|
||||
else:
|
||||
def color(msg, c=None):
|
||||
return msg
|
||||
|
||||
logo = """\
|
||||
'.
|
||||
++++.
|
||||
+++,;+++,
|
||||
:+++ :+++:
|
||||
+++ ,+++;
|
||||
'++; .+++'
|
||||
`+++ `++++
|
||||
+++. `++++
|
||||
;+++ ++++
|
||||
+++ +++
|
||||
+++: '+
|
||||
,+++ +++
|
||||
+++` +++:
|
||||
`+' ,+++
|
||||
`+ + +++
|
||||
`+ +++ '++' :'+++:
|
||||
`+ ++++ `+++ ++++
|
||||
`+ ++++ +++. :+++'
|
||||
`+, ++++ ;+++ +++++
|
||||
`+++, ++++ +++ +++; +
|
||||
,+++, ++++ +++: .+++
|
||||
,+++: '++++++ +++`
|
||||
,+++: '++ '++'
|
||||
,+++: `+++
|
||||
.+++; +++,
|
||||
.+++; ;+++
|
||||
.+++; +++
|
||||
`+++++:
|
||||
`++
|
||||
|
||||
"""
|
||||
|
||||
welcometext = """\
|
||||
For a list of available commands:
|
||||
>>>help()
|
||||
|
||||
To see the documentation for a given command:
|
||||
>>>help("resolve")
|
||||
|
||||
To exit:
|
||||
>>>exit()
|
||||
"""
|
||||
|
||||
welcome = "{:*^60}\n".format(" Welcome to the lbrynet interactive console! ")
|
||||
welcome += "\n".join(["{:<60}".format(w) for w in welcometext.splitlines()])
|
||||
welcome += "\n%s" % ("*" * 60)
|
||||
welcome = color(welcome, "grey")
|
||||
banner = color(logo, "green") + color(welcome, "grey")
|
||||
|
||||
|
||||
def get_methods(daemon):
|
||||
locs = {}
|
||||
|
||||
def wrapped(name, fn):
|
||||
client = LBRYAPIClient.get_client()
|
||||
_fn = getattr(client, name)
|
||||
_fn.__doc__ = fn.__doc__
|
||||
return {name: _fn}
|
||||
|
||||
for method_name, method in daemon.callable_methods.items():
|
||||
locs.update(wrapped(method_name, method))
|
||||
return locs
|
||||
|
||||
|
||||
def run_terminal(callable_methods, started_daemon, quiet=False):
|
||||
locs = {}
|
||||
locs.update(callable_methods)
|
||||
|
||||
def help(method_name=None):
|
||||
if not method_name:
|
||||
print("Available api functions: ")
|
||||
for name in callable_methods:
|
||||
print("\t%s" % name)
|
||||
return
|
||||
if method_name not in callable_methods:
|
||||
print("\"%s\" is not a recognized api function")
|
||||
return
|
||||
print(callable_methods[method_name].__doc__)
|
||||
return
|
||||
|
||||
locs.update({'help': help})
|
||||
|
||||
if started_daemon:
|
||||
def exit(status=None):
|
||||
if not quiet:
|
||||
print("Stopping lbrynet-daemon...")
|
||||
callable_methods['daemon_stop']()
|
||||
return sys.exit(status)
|
||||
|
||||
locs.update({'exit': exit})
|
||||
else:
|
||||
def exit(status=None):
|
||||
try:
|
||||
reactor.callLater(0, reactor.stop)
|
||||
except Exception as err:
|
||||
print("error stopping reactor: {}".format(err))
|
||||
return sys.exit(status)
|
||||
|
||||
locs.update({'exit': exit})
|
||||
|
||||
code.interact(banner if not quiet else "", local=locs)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_server_and_listen(use_auth, analytics_manager, quiet):
|
||||
log_support.configure_console()
|
||||
logging.getLogger("lbrynet").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("lbryum").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("requests").setLevel(logging.CRITICAL)
|
||||
|
||||
analytics_manager.send_server_startup()
|
||||
yield Daemon().start_listening()
|
||||
|
||||
|
||||
def threaded_terminal(started_daemon, quiet):
|
||||
callable_methods = get_methods(Daemon)
|
||||
d = threads.deferToThread(run_terminal, callable_methods, started_daemon, quiet)
|
||||
d.addErrback(lambda err: err.trap(SystemExit))
|
||||
d.addErrback(log.exception)
|
||||
|
||||
|
||||
async def start_lbrynet_console(quiet, use_existing_daemon, useauth):
|
||||
if not utils.check_connection():
|
||||
print("Not connected to internet, unable to start")
|
||||
raise Exception("Not connected to internet, unable to start")
|
||||
if not quiet:
|
||||
print("Starting lbrynet-console...")
|
||||
try:
|
||||
await LBRYAPIClient.get_client().status()
|
||||
d = defer.succeed(False)
|
||||
if not quiet:
|
||||
print("lbrynet-daemon is already running, connecting to it...")
|
||||
except client_exceptions.ClientConnectorError:
|
||||
if not use_existing_daemon:
|
||||
if not quiet:
|
||||
print("Starting lbrynet-daemon...")
|
||||
analytics_manager = analytics.Manager.new_instance()
|
||||
d = start_server_and_listen(useauth, analytics_manager, quiet)
|
||||
else:
|
||||
raise Exception("cannot connect to an existing daemon instance, "
|
||||
"and set to not start a new one")
|
||||
d.addCallback(threaded_terminal, quiet)
|
||||
d.addErrback(log.exception)
|
||||
|
||||
|
||||
def main():
|
||||
conf.initialize_settings()
|
||||
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
|
||||
parser.add_argument(
|
||||
"--use_existing_daemon",
|
||||
help="Start lbrynet-daemon if it isn't already running",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="use_existing_daemon"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--quiet", dest="quiet", action="store_true", default=False
|
||||
)
|
||||
parser.add_argument(
|
||||
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
|
||||
)
|
||||
args = parser.parse_args()
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(start_lbrynet_console(args.quiet, args.use_existing_daemon, args.useauth))
|
||||
reactor.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,74 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows
|
||||
# This needs to happen before anything else
|
||||
if 'win' in sys.platform:
|
||||
import certifi
|
||||
os.environ['SSL_CERT_FILE'] = certifi.where()
|
||||
|
||||
from lbrynet.core import log_support
|
||||
|
||||
import argparse
|
||||
import logging.handlers
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils, system_info
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def test_internet_connection():
|
||||
return utils.check_connection()
|
||||
|
||||
|
||||
def start(argv=None, conf_path=None):
|
||||
if conf_path is not None:
|
||||
conf.conf_file = conf_path
|
||||
|
||||
conf.initialize_settings()
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
|
||||
)
|
||||
parser.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
parser.add_argument(
|
||||
'--version', action="store_true",
|
||||
help='Show daemon version and quit'
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
if args.useauth:
|
||||
conf.settings.update({'use_auth_http': args.useauth}, data_types=(conf.TYPE_CLI,))
|
||||
|
||||
if args.version:
|
||||
version = system_info.get_platform(get_ip=False)
|
||||
version['installation_id'] = conf.settings.installation_id
|
||||
print(utils.json_dumps_pretty(version))
|
||||
return
|
||||
|
||||
lbrynet_log = conf.settings.get_log_filename()
|
||||
log_support.configure_logging(lbrynet_log, not args.quiet, args.verbose)
|
||||
log_support.configure_loggly_handler()
|
||||
log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
|
||||
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
if test_internet_connection():
|
||||
daemon = Daemon()
|
||||
daemon.start_listening()
|
||||
reactor.run()
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
@@ -1 +0,0 @@
|
||||
from . import Components # register Component classes
|
||||
@@ -1,34 +0,0 @@
|
||||
import logging
|
||||
from zope.interface import implementer
|
||||
from twisted.cred import portal, checkers, credentials, error as cred_error
|
||||
from twisted.internet import defer
|
||||
from twisted.web import resource
|
||||
from .keyring import Keyring
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@implementer(portal.IRealm)
|
||||
class HttpPasswordRealm:
|
||||
def __init__(self, resource):
|
||||
self.resource = resource
|
||||
|
||||
def requestAvatar(self, avatarId, mind, *interfaces):
|
||||
log.debug("Processing request for %s", avatarId)
|
||||
if resource.IResource in interfaces:
|
||||
return (resource.IResource, self.resource, lambda: None)
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
@implementer(checkers.ICredentialsChecker)
|
||||
class PasswordChecker:
|
||||
credentialInterfaces = (credentials.IUsernamePassword,)
|
||||
|
||||
def __init__(self, keyring: Keyring):
|
||||
self.api_key = keyring.api_key
|
||||
|
||||
def requestAvatarId(self, creds):
|
||||
if creds.checkPassword(self.api_key.secret.encode()) and creds.username == self.api_key.name.encode():
|
||||
return defer.succeed(creds.username)
|
||||
log.warning('Incorrect username or password')
|
||||
return defer.fail(cred_error.UnauthorizedLogin('Incorrect username or password'))
|
||||
@@ -1,133 +0,0 @@
|
||||
import json
|
||||
import aiohttp
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.daemon.auth.keyring import Keyring, APIKey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
USER_AGENT = "AuthServiceProxy/0.1"
|
||||
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
|
||||
TWISTED_SESSION = "TWISTED_SESSION"
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
HTTP_TIMEOUT = 30
|
||||
|
||||
|
||||
class JSONRPCException(Exception):
|
||||
def __init__(self, rpc_error):
|
||||
super().__init__()
|
||||
self.error = rpc_error
|
||||
|
||||
|
||||
class UnAuthAPIClient:
|
||||
def __init__(self, host, port, session):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.session = session
|
||||
|
||||
def __getattr__(self, method):
|
||||
async def f(*args, **kwargs):
|
||||
return await self.call(method, [args, kwargs])
|
||||
|
||||
return f
|
||||
|
||||
@classmethod
|
||||
async def from_url(cls, url):
|
||||
url_fragment = urlparse(url)
|
||||
host = url_fragment.hostname
|
||||
port = url_fragment.port
|
||||
connector = aiohttp.TCPConnector(
|
||||
ssl=None if not conf.settings['use_https'] else Keyring.load_from_disk().ssl_context
|
||||
)
|
||||
session = aiohttp.ClientSession(connector=connector)
|
||||
return cls(host, port, session)
|
||||
|
||||
async def call(self, method, params=None):
|
||||
message = {'method': method, 'params': params}
|
||||
async with self.session.get(conf.settings.get_api_connection_string(), json=message) as resp:
|
||||
return await resp.json()
|
||||
|
||||
|
||||
class AuthAPIClient:
|
||||
def __init__(self, key, session, cookies, url, login_url):
|
||||
self.session = session
|
||||
self.__api_key = key
|
||||
self.__login_url = login_url
|
||||
self.__id_count = 0
|
||||
self.__url = url
|
||||
self.__cookies = cookies
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name.startswith('__') and name.endswith('__'):
|
||||
raise AttributeError(name)
|
||||
|
||||
def f(*args, **kwargs):
|
||||
return self.call(name, [args, kwargs])
|
||||
|
||||
return f
|
||||
|
||||
async def call(self, method, params=None):
|
||||
params = params or {}
|
||||
self.__id_count += 1
|
||||
|
||||
pre_auth_post_data = {
|
||||
'version': '2',
|
||||
'method': method,
|
||||
'params': params,
|
||||
'id': self.__id_count
|
||||
}
|
||||
to_auth = json.dumps(pre_auth_post_data, sort_keys=True)
|
||||
auth_msg = self.__api_key.get_hmac(to_auth).decode()
|
||||
pre_auth_post_data.update({'hmac': auth_msg})
|
||||
post_data = json.dumps(pre_auth_post_data)
|
||||
|
||||
headers = {
|
||||
'Host': self.__url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'
|
||||
}
|
||||
|
||||
async with self.session.post(self.__login_url, data=post_data, headers=headers) as resp:
|
||||
if resp is None:
|
||||
raise JSONRPCException({'code': -342, 'message': 'missing HTTP response from server'})
|
||||
resp.raise_for_status()
|
||||
|
||||
next_secret = resp.headers.get(LBRY_SECRET, False)
|
||||
if next_secret:
|
||||
self.__api_key.secret = next_secret
|
||||
|
||||
return await resp.json()
|
||||
|
||||
@classmethod
|
||||
async def get_client(cls, key_name=None):
|
||||
api_key_name = key_name or "api"
|
||||
keyring = Keyring.load_from_disk()
|
||||
|
||||
api_key = keyring.api_key
|
||||
login_url = conf.settings.get_api_connection_string(api_key_name, api_key.secret)
|
||||
url = urlparse(login_url)
|
||||
|
||||
headers = {
|
||||
'Host': url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'
|
||||
}
|
||||
connector = aiohttp.TCPConnector(ssl=None if not conf.settings['use_https'] else keyring.ssl_context)
|
||||
session = aiohttp.ClientSession(connector=connector)
|
||||
|
||||
async with session.post(login_url, headers=headers) as r:
|
||||
cookies = r.cookies
|
||||
uid = cookies.get(TWISTED_SECURE_SESSION if conf.settings['use_https'] else TWISTED_SESSION).value
|
||||
api_key = APIKey.create(seed=uid.encode())
|
||||
return cls(api_key, session, cookies, url, login_url)
|
||||
|
||||
|
||||
class LBRYAPIClient:
|
||||
@staticmethod
|
||||
def get_client(conf_path=None):
|
||||
conf.conf_file = conf_path
|
||||
if not conf.settings:
|
||||
conf.initialize_settings()
|
||||
return AuthAPIClient.get_client() if conf.settings['use_auth_http'] else \
|
||||
UnAuthAPIClient.from_url(conf.settings.get_api_connection_string())
|
||||
@@ -1,50 +0,0 @@
|
||||
import logging
|
||||
|
||||
from twisted.web import server, guard, resource
|
||||
from twisted.cred import portal
|
||||
|
||||
from lbrynet import conf
|
||||
from .auth import PasswordChecker, HttpPasswordRealm
|
||||
from ..auth.keyring import Keyring
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HTTPJSONRPCFactory(server.Site):
|
||||
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
|
||||
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
|
||||
self.use_ssl = False
|
||||
|
||||
|
||||
class HTTPSJSONRPCFactory(server.Site):
|
||||
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
|
||||
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
|
||||
self.options = keyring.private_certificate.options()
|
||||
self.use_ssl = True
|
||||
|
||||
|
||||
class AuthJSONRPCResource(resource.Resource):
|
||||
def __init__(self, protocol):
|
||||
resource.Resource.__init__(self)
|
||||
self.putChild(b"", protocol)
|
||||
self.putChild(conf.settings['API_ADDRESS'].encode(), protocol)
|
||||
|
||||
def getChild(self, name, request):
|
||||
request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
|
||||
request.setHeader('expires', '0')
|
||||
return self if name == '' else resource.Resource.getChild(self, name, request)
|
||||
|
||||
def getServerFactory(self, keyring: Keyring, use_authentication: bool, use_https: bool) -> server.Site:
|
||||
factory_class = HTTPSJSONRPCFactory if use_https else HTTPJSONRPCFactory
|
||||
if use_authentication:
|
||||
log.info("Using authenticated API")
|
||||
checker = PasswordChecker(keyring)
|
||||
realm = HttpPasswordRealm(self)
|
||||
portal_to_realm = portal.Portal(realm, [checker, ])
|
||||
root = guard.HTTPAuthSessionWrapper(
|
||||
portal_to_realm, [guard.BasicCredentialFactory('Login to lbrynet api'), ]
|
||||
)
|
||||
else:
|
||||
log.info("Using non-authenticated API")
|
||||
root = self
|
||||
return factory_class(root, keyring)
|
||||
@@ -1,130 +0,0 @@
|
||||
import os
|
||||
import datetime
|
||||
import hmac
|
||||
import hashlib
|
||||
import base58
|
||||
from OpenSSL.crypto import FILETYPE_PEM
|
||||
from ssl import create_default_context, SSLContext
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.x509.name import NameOID, NameAttribute
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from twisted.internet import ssl
|
||||
import logging
|
||||
from lbrynet import conf
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def sha(x: bytes) -> str:
|
||||
h = hashlib.sha256(x).digest()
|
||||
return base58.b58encode(h).decode()
|
||||
|
||||
|
||||
def generate_key(x: bytes = None) -> str:
|
||||
if not x:
|
||||
return sha(os.urandom(256))
|
||||
else:
|
||||
return sha(x)
|
||||
|
||||
|
||||
class APIKey:
|
||||
def __init__(self, secret: str, name: str):
|
||||
self.secret = secret
|
||||
self.name = name
|
||||
|
||||
@classmethod
|
||||
def create(cls, seed=None, name=None):
|
||||
secret = generate_key(seed)
|
||||
return APIKey(secret, name)
|
||||
|
||||
def _raw_key(self) -> str:
|
||||
return base58.b58decode(self.secret)
|
||||
|
||||
def get_hmac(self, message) -> str:
|
||||
decoded_key = self._raw_key()
|
||||
signature = hmac.new(decoded_key, message.encode(), hashlib.sha256)
|
||||
return base58.b58encode(signature.digest())
|
||||
|
||||
def compare_hmac(self, message, token) -> bool:
|
||||
decoded_token = base58.b58decode(token)
|
||||
target = base58.b58decode(self.get_hmac(message))
|
||||
|
||||
try:
|
||||
if len(decoded_token) != len(target):
|
||||
return False
|
||||
return hmac.compare_digest(decoded_token, target)
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
class Keyring:
|
||||
encoding = serialization.Encoding.PEM
|
||||
filetype = FILETYPE_PEM
|
||||
|
||||
def __init__(self, api_key: APIKey, public_certificate: str, private_certificate: ssl.PrivateCertificate = None):
|
||||
self.api_key: APIKey = api_key
|
||||
self.public_certificate: str = public_certificate
|
||||
self.private_certificate: (ssl.PrivateCertificate or None) = private_certificate
|
||||
self.ssl_context: SSLContext = create_default_context(cadata=self.public_certificate)
|
||||
|
||||
@classmethod
|
||||
def load_from_disk(cls):
|
||||
api_key_path = os.path.join(conf.settings['data_dir'], 'auth_token')
|
||||
api_ssl_cert_path = os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem')
|
||||
if not os.path.isfile(api_key_path) or not os.path.isfile(api_ssl_cert_path):
|
||||
return
|
||||
with open(api_key_path, 'rb') as f:
|
||||
api_key = APIKey(f.read().decode(), "api")
|
||||
with open(api_ssl_cert_path, 'rb') as f:
|
||||
public_cert = f.read().decode()
|
||||
return cls(api_key, public_cert)
|
||||
|
||||
@classmethod
|
||||
def generate_and_save(cls):
|
||||
dns = conf.settings['api_host']
|
||||
private_key = rsa.generate_private_key(
|
||||
public_exponent=65537,
|
||||
key_size=4096,
|
||||
backend=default_backend()
|
||||
)
|
||||
subject = issuer = x509.Name([
|
||||
NameAttribute(NameOID.COUNTRY_NAME, "US"),
|
||||
NameAttribute(NameOID.ORGANIZATION_NAME, "LBRY"),
|
||||
NameAttribute(NameOID.COMMON_NAME, "LBRY API"),
|
||||
])
|
||||
alternative_name = x509.SubjectAlternativeName([x509.DNSName(dns)])
|
||||
certificate = x509.CertificateBuilder(
|
||||
subject_name=subject,
|
||||
issuer_name=issuer,
|
||||
public_key=private_key.public_key(),
|
||||
serial_number=x509.random_serial_number(),
|
||||
not_valid_before=datetime.datetime.utcnow(),
|
||||
not_valid_after=datetime.datetime.utcnow() + datetime.timedelta(days=365),
|
||||
extensions=[x509.Extension(oid=alternative_name.oid, critical=False, value=alternative_name)]
|
||||
).sign(private_key, hashes.SHA256(), default_backend())
|
||||
public_certificate = certificate.public_bytes(cls.encoding).decode()
|
||||
private_certificate = ssl.PrivateCertificate.load(
|
||||
public_certificate,
|
||||
ssl.KeyPair.load(
|
||||
private_key.private_bytes(
|
||||
encoding=cls.encoding,
|
||||
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
encryption_algorithm=serialization.NoEncryption()
|
||||
).decode(),
|
||||
cls.filetype
|
||||
),
|
||||
cls.filetype
|
||||
)
|
||||
|
||||
auth_token = APIKey.create(seed=None, name="api")
|
||||
|
||||
with open(os.path.join(conf.settings['data_dir'], 'auth_token'), 'wb') as f:
|
||||
f.write(auth_token.secret.encode())
|
||||
|
||||
with open(os.path.join(conf.settings['data_dir'], 'api_ssl_cert.pem'), 'wb') as f:
|
||||
f.write(public_certificate.encode())
|
||||
|
||||
return cls(auth_token, public_certificate, private_certificate)
|
||||
@@ -1,598 +0,0 @@
|
||||
import logging
|
||||
from six.moves.urllib import parse as urlparse
|
||||
import json
|
||||
import inspect
|
||||
import signal
|
||||
|
||||
from functools import wraps
|
||||
from twisted.web import server
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet.error import ConnectionDone, ConnectionLost
|
||||
from txjsonrpc import jsonrpclib
|
||||
from traceback import format_exc
|
||||
|
||||
from lbrynet import conf, analytics
|
||||
from lbrynet.core.Error import InvalidAuthenticationToken
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core.Error import ComponentsNotStarted, ComponentStartConditionNotMet
|
||||
from lbrynet.core.looping_call_manager import LoopingCallManager
|
||||
from lbrynet.daemon.ComponentManager import ComponentManager
|
||||
from .keyring import APIKey, Keyring
|
||||
from .undecorated import undecorated
|
||||
from .factory import AuthJSONRPCResource
|
||||
from lbrynet.daemon.json_response_encoder import JSONResponseEncoder
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
EMPTY_PARAMS = [{}]
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
|
||||
|
||||
class JSONRPCError:
|
||||
# http://www.jsonrpc.org/specification#error_object
|
||||
CODE_PARSE_ERROR = -32700 # Invalid JSON. Error while parsing the JSON text.
|
||||
CODE_INVALID_REQUEST = -32600 # The JSON sent is not a valid Request object.
|
||||
CODE_METHOD_NOT_FOUND = -32601 # The method does not exist / is not available.
|
||||
CODE_INVALID_PARAMS = -32602 # Invalid method parameter(s).
|
||||
CODE_INTERNAL_ERROR = -32603 # Internal JSON-RPC error (I think this is like a 500?)
|
||||
CODE_APPLICATION_ERROR = -32500 # Generic error with our app??
|
||||
CODE_AUTHENTICATION_ERROR = -32501 # Authentication failed
|
||||
|
||||
MESSAGES = {
|
||||
CODE_PARSE_ERROR: "Parse Error. Data is not valid JSON.",
|
||||
CODE_INVALID_REQUEST: "JSON data is not a valid Request",
|
||||
CODE_METHOD_NOT_FOUND: "Method Not Found",
|
||||
CODE_INVALID_PARAMS: "Invalid Params",
|
||||
CODE_INTERNAL_ERROR: "Internal Error",
|
||||
CODE_AUTHENTICATION_ERROR: "Authentication Failed",
|
||||
}
|
||||
|
||||
HTTP_CODES = {
|
||||
CODE_INVALID_REQUEST: 400,
|
||||
CODE_PARSE_ERROR: 400,
|
||||
CODE_INVALID_PARAMS: 400,
|
||||
CODE_METHOD_NOT_FOUND: 404,
|
||||
CODE_INTERNAL_ERROR: 500,
|
||||
CODE_APPLICATION_ERROR: 500,
|
||||
CODE_AUTHENTICATION_ERROR: 401,
|
||||
}
|
||||
|
||||
def __init__(self, message, code=CODE_APPLICATION_ERROR, traceback=None, data=None):
|
||||
assert isinstance(code, int), "'code' must be an int"
|
||||
assert (data is None or isinstance(data, dict)), "'data' must be None or a dict"
|
||||
self.code = code
|
||||
if message is None:
|
||||
message = self.MESSAGES[code] if code in self.MESSAGES else "API Error"
|
||||
self.message = message
|
||||
self.data = {} if data is None else data
|
||||
self.traceback = []
|
||||
if traceback is not None:
|
||||
trace_lines = traceback.split("\n")
|
||||
for i, t in enumerate(trace_lines):
|
||||
if "--- <exception caught here> ---" in t:
|
||||
if len(trace_lines) > i + 1:
|
||||
self.traceback = [j for j in trace_lines[i+1:] if j]
|
||||
break
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'code': self.code,
|
||||
'message': self.message,
|
||||
'data': self.traceback
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def create_from_exception(cls, message, code=CODE_APPLICATION_ERROR, traceback=None):
|
||||
return cls(message, code=code, traceback=traceback)
|
||||
|
||||
|
||||
class UnknownAPIMethodError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def trap(err, *to_trap):
|
||||
err.trap(*to_trap)
|
||||
|
||||
|
||||
def jsonrpc_dumps_pretty(obj, **kwargs):
|
||||
try:
|
||||
id_ = kwargs.pop("id")
|
||||
except KeyError:
|
||||
id_ = None
|
||||
|
||||
if isinstance(obj, JSONRPCError):
|
||||
data = {"jsonrpc": "2.0", "error": obj.to_dict(), "id": id_}
|
||||
else:
|
||||
data = {"jsonrpc": "2.0", "result": obj, "id": id_}
|
||||
|
||||
return json.dumps(data, cls=JSONResponseEncoder, sort_keys=True, indent=2, **kwargs) + "\n"
|
||||
|
||||
|
||||
class JSONRPCServerType(type):
|
||||
def __new__(mcs, name, bases, newattrs):
|
||||
klass = type.__new__(mcs, name, bases, newattrs)
|
||||
klass.callable_methods = {}
|
||||
klass.deprecated_methods = {}
|
||||
|
||||
for methodname in dir(klass):
|
||||
if methodname.startswith("jsonrpc_"):
|
||||
method = getattr(klass, methodname)
|
||||
if not hasattr(method, '_deprecated'):
|
||||
klass.callable_methods.update({methodname.split("jsonrpc_")[1]: method})
|
||||
else:
|
||||
klass.deprecated_methods.update({methodname.split("jsonrpc_")[1]: method})
|
||||
return klass
|
||||
|
||||
|
||||
class AuthorizedBase(metaclass=JSONRPCServerType):
|
||||
|
||||
@staticmethod
|
||||
def deprecated(new_command=None):
|
||||
def _deprecated_wrapper(f):
|
||||
f.new_command = new_command
|
||||
f._deprecated = True
|
||||
return f
|
||||
return _deprecated_wrapper
|
||||
|
||||
@staticmethod
|
||||
def requires(*components, **conditions):
|
||||
if conditions and ["conditions"] != list(conditions.keys()):
|
||||
raise SyntaxError("invalid conditions argument")
|
||||
condition_names = conditions.get("conditions", [])
|
||||
|
||||
def _wrap(fn):
|
||||
@defer.inlineCallbacks
|
||||
@wraps(fn)
|
||||
def _inner(*args, **kwargs):
|
||||
component_manager = args[0].component_manager
|
||||
for condition_name in condition_names:
|
||||
condition_result, err_msg = yield component_manager.evaluate_condition(condition_name)
|
||||
if not condition_result:
|
||||
raise ComponentStartConditionNotMet(err_msg)
|
||||
if not component_manager.all_components_running(*components):
|
||||
raise ComponentsNotStarted("the following required components have not yet started: "
|
||||
"%s" % json.dumps(components))
|
||||
result = yield fn(*args, **kwargs)
|
||||
defer.returnValue(result)
|
||||
return _inner
|
||||
return _wrap
|
||||
|
||||
|
||||
class AuthJSONRPCServer(AuthorizedBase):
|
||||
"""
|
||||
Authorized JSONRPC server used as the base class for the LBRY API
|
||||
|
||||
API methods are named with a leading "jsonrpc_"
|
||||
|
||||
Attributes:
|
||||
sessions (dict): (dict): {<session id>: <lbrynet.daemon.auth.util.APIKey>}
|
||||
callable_methods (dict): {<api method name>: <api method>}
|
||||
|
||||
Authentication:
|
||||
If use_authentication is true, basic HTTP and HMAC authentication will be used for all requests and the
|
||||
service url will require a username and password.
|
||||
|
||||
To start an authenticated session a client sends an HTTP POST to <user>:<password>@<api host>:<api port>.
|
||||
If accepted, the server replies with a TWISTED_SESSION cookie containing a session id and the message "OK".
|
||||
The client initializes their shared secret for hmac to be the b64 encoded sha256 of their session id.
|
||||
|
||||
To send an authenticated request a client sends an HTTP POST to the auth api url with the TWISTED_SESSION
|
||||
cookie and includes a hmac token in the message using the previously set shared secret. If the token is valid
|
||||
the server will randomize the shared secret and return the new value under the LBRY_SECRET header, which the
|
||||
client uses to generate the token for their next request.
|
||||
"""
|
||||
#implements(resource.IResource)
|
||||
|
||||
isLeaf = True
|
||||
allowed_during_startup = []
|
||||
component_attributes = {}
|
||||
|
||||
def __init__(self, analytics_manager=None, component_manager=None, use_authentication=None, use_https=None,
|
||||
to_skip=None, looping_calls=None, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.analytics_manager = analytics_manager or analytics.Manager.new_instance()
|
||||
self.component_manager = component_manager or ComponentManager(
|
||||
analytics_manager=self.analytics_manager,
|
||||
skip_components=to_skip or [],
|
||||
reactor=reactor
|
||||
)
|
||||
self.looping_call_manager = LoopingCallManager({n: lc for n, (lc, t) in (looping_calls or {}).items()})
|
||||
self._looping_call_times = {n: t for n, (lc, t) in (looping_calls or {}).items()}
|
||||
self._use_authentication = use_authentication or conf.settings['use_auth_http']
|
||||
self._use_https = use_https or conf.settings['use_https']
|
||||
self.listening_port = None
|
||||
self._component_setup_deferred = None
|
||||
self.announced_startup = False
|
||||
self.sessions = {}
|
||||
self.server = None
|
||||
self.keyring = Keyring.generate_and_save()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_listening(self):
|
||||
from twisted.internet import reactor, error as tx_error
|
||||
|
||||
try:
|
||||
self.server = self.get_server_factory()
|
||||
if self.server.use_ssl:
|
||||
log.info("Using SSL")
|
||||
self.listening_port = reactor.listenSSL(
|
||||
conf.settings['api_port'], self.server, self.server.options, interface=conf.settings['api_host']
|
||||
)
|
||||
else:
|
||||
log.info("Not using SSL")
|
||||
self.listening_port = reactor.listenTCP(
|
||||
conf.settings['api_port'], self.server, interface=conf.settings['api_host']
|
||||
)
|
||||
log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
|
||||
yield self.setup()
|
||||
self.analytics_manager.send_server_startup_success()
|
||||
except tx_error.CannotListenError:
|
||||
log.error('lbrynet API failed to bind TCP %s:%i for listening. Daemon is already running or this port is '
|
||||
'already in use by another application.', conf.settings['api_host'], conf.settings['api_port'])
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
except defer.CancelledError:
|
||||
log.info("shutting down before finished starting")
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
except Exception as err:
|
||||
self.analytics_manager.send_server_startup_error(str(err))
|
||||
log.exception('Failed to start lbrynet-daemon')
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
|
||||
def setup(self):
|
||||
from twisted.internet import reactor
|
||||
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
|
||||
if not self.analytics_manager.is_started:
|
||||
self.analytics_manager.start()
|
||||
self.analytics_manager.send_server_startup()
|
||||
for lc_name, lc_time in self._looping_call_times.items():
|
||||
self.looping_call_manager.start(lc_name, lc_time)
|
||||
|
||||
def update_attribute(setup_result, component):
|
||||
setattr(self, self.component_attributes[component.component_name], component.component)
|
||||
|
||||
kwargs = {component: update_attribute for component in self.component_attributes.keys()}
|
||||
self._component_setup_deferred = self.component_manager.setup(**kwargs)
|
||||
return self._component_setup_deferred
|
||||
|
||||
@staticmethod
|
||||
def _already_shutting_down(sig_num, frame):
|
||||
log.info("Already shutting down")
|
||||
|
||||
def _shutdown(self):
|
||||
# ignore INT/TERM signals once shutdown has started
|
||||
signal.signal(signal.SIGINT, self._already_shutting_down)
|
||||
signal.signal(signal.SIGTERM, self._already_shutting_down)
|
||||
if self.listening_port:
|
||||
self.listening_port.stopListening()
|
||||
self.looping_call_manager.shutdown()
|
||||
if self.server is not None:
|
||||
for session in list(self.server.sessions.values()):
|
||||
session.expire()
|
||||
if self.analytics_manager:
|
||||
self.analytics_manager.shutdown()
|
||||
try:
|
||||
self._component_setup_deferred.cancel()
|
||||
except (AttributeError, defer.CancelledError):
|
||||
pass
|
||||
if self.component_manager is not None:
|
||||
d = self.component_manager.stop()
|
||||
d.addErrback(log.fail(), 'Failure while shutting down')
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
return d
|
||||
|
||||
def get_server_factory(self):
|
||||
return AuthJSONRPCResource(self).getServerFactory(self.keyring, self._use_authentication, self._use_https)
|
||||
|
||||
def _set_headers(self, request, data, update_secret=False):
|
||||
if conf.settings['allowed_origin']:
|
||||
request.setHeader("Access-Control-Allow-Origin", conf.settings['allowed_origin'])
|
||||
request.setHeader("Content-Type", "application/json")
|
||||
request.setHeader("Accept", "application/json-rpc")
|
||||
request.setHeader("Content-Length", str(len(data)))
|
||||
if update_secret:
|
||||
session_id = request.getSession().uid
|
||||
request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret)
|
||||
|
||||
@staticmethod
|
||||
def _render_message(request, message: str):
|
||||
request.write(message.encode())
|
||||
request.finish()
|
||||
|
||||
def _render_error(self, failure, request, id_):
|
||||
if isinstance(failure, JSONRPCError):
|
||||
error = failure
|
||||
elif isinstance(failure, Failure):
|
||||
# maybe failure is JSONRPCError wrapped in a twisted Failure
|
||||
error = failure.check(JSONRPCError)
|
||||
if error is None:
|
||||
# maybe its a twisted Failure with another type of error
|
||||
if hasattr(failure.type, "code"):
|
||||
error_code = failure.type.code
|
||||
else:
|
||||
error_code = JSONRPCError.CODE_APPLICATION_ERROR
|
||||
error = JSONRPCError.create_from_exception(
|
||||
failure.getErrorMessage() or failure.type.__name__,
|
||||
code=error_code,
|
||||
traceback=failure.getTraceback()
|
||||
)
|
||||
if not failure.check(ComponentsNotStarted, ComponentStartConditionNotMet):
|
||||
log.warning("error processing api request: %s\ntraceback: %s", error.message,
|
||||
"\n".join(error.traceback))
|
||||
else:
|
||||
# last resort, just cast it as a string
|
||||
error = JSONRPCError(str(failure))
|
||||
|
||||
response_content = jsonrpc_dumps_pretty(error, id=id_, ledger=self.ledger)
|
||||
self._set_headers(request, response_content)
|
||||
request.setResponseCode(200)
|
||||
self._render_message(request, response_content)
|
||||
|
||||
@staticmethod
|
||||
def _handle_dropped_request(result, d, function_name):
|
||||
if not d.called:
|
||||
log.warning("Cancelling dropped api request %s", function_name)
|
||||
d.cancel()
|
||||
|
||||
def render(self, request):
|
||||
try:
|
||||
return self._render(request)
|
||||
except BaseException as e:
|
||||
log.error(e)
|
||||
error = JSONRPCError.create_from_exception(str(e), traceback=format_exc())
|
||||
self._render_error(error, request, None)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def _render(self, request):
|
||||
time_in = utils.now()
|
||||
# if not self._check_headers(request):
|
||||
# self._render_error(Failure(InvalidHeaderError()), request, None)
|
||||
# return server.NOT_DONE_YET
|
||||
session = request.getSession()
|
||||
session_id = session.uid
|
||||
finished_deferred = request.notifyFinish()
|
||||
|
||||
if self._use_authentication:
|
||||
# if this is a new session, send a new secret and set the expiration
|
||||
# otherwise, session.touch()
|
||||
if self._initialize_session(session_id):
|
||||
def expire_session():
|
||||
self._unregister_user_session(session_id)
|
||||
|
||||
session.notifyOnExpire(expire_session)
|
||||
message = "OK"
|
||||
request.setResponseCode(200)
|
||||
self._set_headers(request, message, True)
|
||||
self._render_message(request, message)
|
||||
return server.NOT_DONE_YET
|
||||
else:
|
||||
session.touch()
|
||||
|
||||
request.content.seek(0, 0)
|
||||
content = request.content.read().decode()
|
||||
try:
|
||||
parsed = jsonrpclib.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
log.warning("Unable to decode request json")
|
||||
self._render_error(JSONRPCError(None, code=JSONRPCError.CODE_PARSE_ERROR), request, None)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
request_id = None
|
||||
try:
|
||||
function_name = parsed.get('method')
|
||||
args = parsed.get('params', {})
|
||||
request_id = parsed.get('id', None)
|
||||
token = parsed.pop('hmac', None)
|
||||
except AttributeError as err:
|
||||
log.warning(err)
|
||||
self._render_error(
|
||||
JSONRPCError(None, code=JSONRPCError.CODE_INVALID_REQUEST), request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
reply_with_next_secret = False
|
||||
if self._use_authentication:
|
||||
try:
|
||||
self._verify_token(session_id, parsed, token)
|
||||
except InvalidAuthenticationToken as err:
|
||||
log.warning("API validation failed")
|
||||
self._render_error(
|
||||
JSONRPCError.create_from_exception(
|
||||
str(err),
|
||||
code=JSONRPCError.CODE_AUTHENTICATION_ERROR,
|
||||
traceback=format_exc()
|
||||
),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
request.addCookie("TWISTED_SESSION", session_id)
|
||||
self._update_session_secret(session_id)
|
||||
reply_with_next_secret = True
|
||||
|
||||
try:
|
||||
fn = self._get_jsonrpc_method(function_name)
|
||||
except UnknownAPIMethodError as err:
|
||||
log.warning('Failed to get function %s: %s', function_name, err)
|
||||
self._render_error(
|
||||
JSONRPCError(None, code=JSONRPCError.CODE_METHOD_NOT_FOUND),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
if args in (EMPTY_PARAMS, []):
|
||||
_args, _kwargs = (), {}
|
||||
elif isinstance(args, dict):
|
||||
_args, _kwargs = (), args
|
||||
elif len(args) == 1 and isinstance(args[0], dict):
|
||||
# TODO: this is for backwards compatibility. Remove this once API and UI are updated
|
||||
# TODO: also delete EMPTY_PARAMS then
|
||||
_args, _kwargs = (), args[0]
|
||||
elif len(args) == 2 and isinstance(args[0], list) and isinstance(args[1], dict):
|
||||
_args, _kwargs = args
|
||||
else:
|
||||
raise ValueError('invalid args format')
|
||||
|
||||
params_error, erroneous_params = self._check_params(fn, _args, _kwargs)
|
||||
if params_error is not None:
|
||||
params_error_message = '{} for {} command: {}'.format(
|
||||
params_error, function_name, ', '.join(erroneous_params)
|
||||
)
|
||||
log.warning(params_error_message)
|
||||
self._render_error(
|
||||
JSONRPCError(params_error_message, code=JSONRPCError.CODE_INVALID_PARAMS),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
d = defer.maybeDeferred(fn, self, *_args, **_kwargs)
|
||||
|
||||
# finished_deferred will callback when the request is finished
|
||||
# and errback if something went wrong. If the errback is
|
||||
# called, cancel the deferred stack. This is to prevent
|
||||
# request.finish() from being called on a closed request.
|
||||
finished_deferred.addErrback(self._handle_dropped_request, d, function_name)
|
||||
|
||||
d.addCallback(self._callback_render, request, request_id, reply_with_next_secret)
|
||||
d.addErrback(trap, ConnectionDone, ConnectionLost, defer.CancelledError)
|
||||
d.addErrback(self._render_error, request, request_id)
|
||||
d.addBoth(lambda _: log.debug("%s took %f",
|
||||
function_name,
|
||||
(utils.now() - time_in).total_seconds()))
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def _register_user_session(self, session_id):
|
||||
"""
|
||||
Add or update a HMAC secret for a session
|
||||
|
||||
@param session_id:
|
||||
@return: secret
|
||||
"""
|
||||
log.info("Started new api session")
|
||||
token = APIKey.create(seed=session_id)
|
||||
self.sessions.update({session_id: token})
|
||||
|
||||
def _unregister_user_session(self, session_id):
|
||||
log.info("Unregister API session")
|
||||
del self.sessions[session_id]
|
||||
|
||||
def _check_headers(self, request):
|
||||
return (
|
||||
self._check_header_source(request, 'Origin') and
|
||||
self._check_header_source(request, 'Referer')
|
||||
)
|
||||
|
||||
def _check_header_source(self, request, header):
|
||||
"""Check if the source of the request is allowed based on the header value."""
|
||||
source = request.getHeader(header)
|
||||
if not self._check_source_of_request(source):
|
||||
log.warning("Attempted api call from invalid %s: %s", header, source)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _check_source_of_request(self, source):
|
||||
if source is None:
|
||||
return True
|
||||
if conf.settings['api_host'] == '0.0.0.0':
|
||||
return True
|
||||
server, port = self.get_server_port(source)
|
||||
return self._check_server_port(server, port)
|
||||
|
||||
def _check_server_port(self, server, port):
|
||||
api = (conf.settings['api_host'], conf.settings['api_port'])
|
||||
return (server, port) == api or self._is_from_allowed_origin(server, port)
|
||||
|
||||
def _is_from_allowed_origin(self, server, port):
|
||||
allowed_origin = conf.settings['allowed_origin']
|
||||
if not allowed_origin:
|
||||
return False
|
||||
if allowed_origin == '*':
|
||||
return True
|
||||
allowed_server, allowed_port = self.get_server_port(allowed_origin)
|
||||
return (allowed_server, allowed_port) == (server, port)
|
||||
|
||||
def get_server_port(self, origin):
|
||||
parsed = urlparse.urlparse(origin)
|
||||
server_port = parsed.netloc.split(':')
|
||||
assert len(server_port) <= 2
|
||||
if len(server_port) == 2:
|
||||
return server_port[0], int(server_port[1])
|
||||
else:
|
||||
return server_port[0], 80
|
||||
|
||||
def _verify_method_is_callable(self, function_path):
|
||||
if function_path not in self.callable_methods:
|
||||
raise UnknownAPIMethodError(function_path)
|
||||
|
||||
def _get_jsonrpc_method(self, function_path):
|
||||
if function_path in self.deprecated_methods:
|
||||
new_command = self.deprecated_methods[function_path].new_command
|
||||
log.warning('API function \"%s\" is deprecated, please update to use \"%s\"',
|
||||
function_path, new_command)
|
||||
function_path = new_command
|
||||
self._verify_method_is_callable(function_path)
|
||||
return self.callable_methods.get(function_path)
|
||||
|
||||
@staticmethod
|
||||
def _check_params(function, args_tup, args_dict):
|
||||
argspec = inspect.getfullargspec(undecorated(function))
|
||||
num_optional_params = 0 if argspec.defaults is None else len(argspec.defaults)
|
||||
|
||||
duplicate_params = [
|
||||
duplicate_param
|
||||
for duplicate_param in argspec.args[1:len(args_tup) + 1]
|
||||
if duplicate_param in args_dict
|
||||
]
|
||||
|
||||
if duplicate_params:
|
||||
return 'Duplicate parameters', duplicate_params
|
||||
|
||||
missing_required_params = [
|
||||
required_param
|
||||
for required_param in argspec.args[len(args_tup)+1:-num_optional_params]
|
||||
if required_param not in args_dict
|
||||
]
|
||||
if len(missing_required_params):
|
||||
return 'Missing required parameters', missing_required_params
|
||||
|
||||
extraneous_params = [] if argspec.varkw is not None else [
|
||||
extra_param
|
||||
for extra_param in args_dict
|
||||
if extra_param not in argspec.args[1:]
|
||||
]
|
||||
if len(extraneous_params):
|
||||
return 'Extraneous parameters', extraneous_params
|
||||
|
||||
return None, None
|
||||
|
||||
def _initialize_session(self, session_id):
|
||||
if not self.sessions.get(session_id):
|
||||
self._register_user_session(session_id)
|
||||
return True
|
||||
return False
|
||||
|
||||
def _verify_token(self, session_id, message, token):
|
||||
if token is None:
|
||||
raise InvalidAuthenticationToken('Authentication token not found')
|
||||
to_auth = json.dumps(message, sort_keys=True)
|
||||
api_key = self.sessions.get(session_id)
|
||||
if not api_key.compare_hmac(to_auth, token):
|
||||
raise InvalidAuthenticationToken('Invalid authentication token')
|
||||
|
||||
def _update_session_secret(self, session_id):
|
||||
self.sessions.update({session_id: APIKey.create(name=session_id)})
|
||||
|
||||
def _callback_render(self, result, request, id_, auth_required=False):
|
||||
try:
|
||||
message = jsonrpc_dumps_pretty(result, id=id_, ledger=self.ledger)
|
||||
request.setResponseCode(200)
|
||||
self._set_headers(request, message, auth_required)
|
||||
self._render_message(request, message)
|
||||
except Exception as err:
|
||||
log.exception("Failed to render API response: %s", result)
|
||||
self._render_error(err, request, id_)
|
||||
|
||||
@staticmethod
|
||||
def _render_response(result):
|
||||
return defer.succeed(result)
|
||||
@@ -1,57 +0,0 @@
|
||||
from decimal import Decimal
|
||||
from binascii import hexlify
|
||||
from datetime import datetime
|
||||
from json import JSONEncoder
|
||||
from lbrynet.wallet.transaction import Transaction, Output
|
||||
|
||||
|
||||
class JSONResponseEncoder(JSONEncoder):
|
||||
|
||||
def __init__(self, *args, ledger, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ledger = ledger
|
||||
|
||||
def default(self, obj): # pylint: disable=method-hidden
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
if isinstance(obj, Output):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
if isinstance(obj, bytes):
|
||||
return obj.decode()
|
||||
return super().default(obj)
|
||||
|
||||
def encode_transaction(self, tx):
|
||||
return {
|
||||
'txid': tx.id,
|
||||
'height': tx.height,
|
||||
'inputs': [self.encode_input(txo) for txo in tx.inputs],
|
||||
'outputs': [self.encode_output(txo) for txo in tx.outputs],
|
||||
'total_input': tx.input_sum,
|
||||
'total_output': tx.input_sum - tx.fee,
|
||||
'total_fee': tx.fee,
|
||||
'hex': hexlify(tx.raw).decode(),
|
||||
}
|
||||
|
||||
def encode_output(self, txo):
|
||||
output = {
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'amount': txo.amount,
|
||||
'address': txo.get_address(self.ledger),
|
||||
'is_claim': txo.script.is_claim_name,
|
||||
'is_support': txo.script.is_support_claim,
|
||||
'is_update': txo.script.is_update_claim,
|
||||
}
|
||||
if txo.is_change is not None:
|
||||
output['is_change'] = txo.is_change
|
||||
return output
|
||||
|
||||
def encode_input(self, txi):
|
||||
return self.encode_output(txi.txo_ref.txo) if txi.txo_ref.txo is not None else {
|
||||
'txid': txi.txo_ref.tx_ref.id,
|
||||
'nout': txi.txo_ref.position
|
||||
}
|
||||
@@ -40,7 +40,7 @@ class _Contact:
|
||||
self._token = (None, 0) # token, timestamp
|
||||
|
||||
def update_token(self, token):
|
||||
self._token = token, self.getTime()
|
||||
self._token = token, self.getTime() if token else 0
|
||||
|
||||
@property
|
||||
def token(self):
|
||||
@@ -67,7 +67,7 @@ class _Contact:
|
||||
|
||||
@property
|
||||
def lastFailed(self):
|
||||
return self._contactManager._rpc_failures.get((self.address, self.port), [None])[-1]
|
||||
return (self.failures or [None])[-1]
|
||||
|
||||
@property
|
||||
def failures(self):
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from collections import UserDict
|
||||
from . import constants
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class DictDataStore(UserDict):
|
||||
@@ -44,7 +44,7 @@ class DictDataStore(UserDict):
|
||||
del self[key]
|
||||
|
||||
def hasPeersForBlob(self, key):
|
||||
return True if key in self and len(tuple(self.filter_bad_and_expired_peers(key))) else False
|
||||
return bool(key in self and len(tuple(self.filter_bad_and_expired_peers(key))))
|
||||
|
||||
def addPeerToBlob(self, contact, key, compact_address, lastPublished, originallyPublished, originalPublisherID):
|
||||
if key in self:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from .error import DecodeError
|
||||
from lbrynet.dht.error import DecodeError
|
||||
|
||||
|
||||
def bencode(data):
|
||||
|
||||
@@ -13,14 +13,12 @@ class DecodeError(Exception):
|
||||
Should be raised by an C{Encoding} implementation if decode operation
|
||||
fails
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class BucketFull(Exception):
|
||||
"""
|
||||
Raised when the bucket is full
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class UnknownRemoteException(Exception):
|
||||
|
||||
@@ -10,7 +10,6 @@ class IDataStore(Interface):
|
||||
|
||||
def keys(self):
|
||||
""" Return a list of the keys in this data store """
|
||||
pass
|
||||
|
||||
def removeExpiredPeers(self):
|
||||
pass
|
||||
@@ -59,7 +58,7 @@ class IRoutingTable(Interface):
|
||||
@param count: the amount of contacts to return
|
||||
@type count: int
|
||||
@param _rpcNodeID: Used during RPC, this is be the sender's Node ID
|
||||
Whatever ID is passed in the paramater will get
|
||||
Whatever ID is passed in the parameter will get
|
||||
excluded from the list of returned contacts.
|
||||
@type _rpcNodeID: str
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from .distance import Distance
|
||||
from .error import TimeoutError
|
||||
from . import constants
|
||||
from lbrynet.dht.distance import Distance
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht import constants
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -67,7 +67,7 @@ class _IterativeFind:
|
||||
|
||||
def getContactTriples(self, result):
|
||||
if self.is_find_value_request:
|
||||
contact_triples = result['contacts']
|
||||
contact_triples = result[b'contacts']
|
||||
else:
|
||||
contact_triples = result
|
||||
for contact_tup in contact_triples:
|
||||
@@ -112,11 +112,11 @@ class _IterativeFind:
|
||||
# We are looking for a value, and the remote node didn't have it
|
||||
# - mark it as the closest "empty" node, if it is
|
||||
# TODO: store to this peer after finding the value as per the kademlia spec
|
||||
if 'closestNodeNoValue' in self.find_value_result:
|
||||
if b'closestNodeNoValue' in self.find_value_result:
|
||||
if self.is_closer(contact):
|
||||
self.find_value_result['closestNodeNoValue'] = contact
|
||||
self.find_value_result[b'closestNodeNoValue'] = contact
|
||||
else:
|
||||
self.find_value_result['closestNodeNoValue'] = contact
|
||||
self.find_value_result[b'closestNodeNoValue'] = contact
|
||||
contactTriples = self.getContactTriples(result)
|
||||
for contactTriple in contactTriples:
|
||||
if (contactTriple[1], contactTriple[2]) in ((c.address, c.port) for c in self.already_contacted):
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import logging
|
||||
|
||||
from . import constants
|
||||
from .distance import Distance
|
||||
from .error import BucketFull
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.distance import Distance
|
||||
from lbrynet.dht.error import BucketFull
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
from . import msgtypes
|
||||
from lbrynet.dht import msgtypes
|
||||
|
||||
|
||||
class MessageTranslator:
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
from lbrynet.core.utils import generate_id
|
||||
from . import constants
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class Message:
|
||||
@@ -48,5 +48,5 @@ class ErrorMessage(ResponseMessage):
|
||||
def __init__(self, rpcID, nodeID, exceptionType, errorMessage):
|
||||
super().__init__(rpcID, nodeID, errorMessage)
|
||||
if isinstance(exceptionType, type):
|
||||
exceptionType = ('%s.%s' % (exceptionType.__module__, exceptionType.__name__)).encode()
|
||||
exceptionType = (f'{exceptionType.__module__}.{exceptionType.__name__}').encode()
|
||||
self.exceptionType = exceptionType
|
||||
@@ -5,17 +5,12 @@ from functools import reduce
|
||||
|
||||
from twisted.internet import defer, error, task
|
||||
|
||||
from lbrynet.core.utils import generate_id, DeferredDict
|
||||
from lbrynet.core.call_later_manager import CallLaterManager
|
||||
from lbrynet.core.PeerManager import PeerManager
|
||||
from .error import TimeoutError
|
||||
from . import constants
|
||||
from . import routingtable
|
||||
from . import datastore
|
||||
from . import protocol
|
||||
from .peerfinder import DHTPeerFinder
|
||||
from .contact import ContactManager
|
||||
from .iterativefind import iterativeFind
|
||||
from lbrynet.utils import generate_id, DeferredDict
|
||||
from lbrynet.dht.call_later_manager import CallLaterManager
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht import constants, routingtable, datastore, protocol
|
||||
from lbrynet.dht.contact import ContactManager
|
||||
from lbrynet.dht.iterativefind import iterativeFind
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -83,8 +78,8 @@ class Node(MockKademliaHelper):
|
||||
def __init__(self, node_id=None, udpPort=4000, dataStore=None,
|
||||
routingTableClass=None, networkProtocol=None,
|
||||
externalIP=None, peerPort=3333, listenUDP=None,
|
||||
callLater=None, resolve=None, clock=None, peer_finder=None,
|
||||
peer_manager=None, interface='', externalUDPPort=None):
|
||||
callLater=None, resolve=None, clock=None,
|
||||
interface='', externalUDPPort=None):
|
||||
"""
|
||||
@param dataStore: The data store to use. This must be class inheriting
|
||||
from the C{DataStore} interface (or providing the
|
||||
@@ -124,20 +119,13 @@ class Node(MockKademliaHelper):
|
||||
else:
|
||||
self._routingTable = routingTableClass(self.node_id, self.clock.seconds)
|
||||
|
||||
# Initialize this node's network access mechanisms
|
||||
if networkProtocol is None:
|
||||
self._protocol = protocol.KademliaProtocol(self)
|
||||
else:
|
||||
self._protocol = networkProtocol
|
||||
# Initialize the data storage mechanism used by this node
|
||||
self._protocol = networkProtocol or protocol.KademliaProtocol(self)
|
||||
self.token_secret = self._generateID()
|
||||
self.old_token_secret = None
|
||||
self.externalIP = externalIP
|
||||
self.peerPort = peerPort
|
||||
self.externalUDPPort = externalUDPPort or self.port
|
||||
self._dataStore = dataStore or datastore.DictDataStore(self.clock.seconds)
|
||||
self.peer_manager = peer_manager or PeerManager()
|
||||
self.peer_finder = peer_finder or DHTPeerFinder(self, self.peer_manager)
|
||||
self._join_deferred = None
|
||||
|
||||
#def __del__(self):
|
||||
@@ -220,7 +208,7 @@ class Node(MockKademliaHelper):
|
||||
if not bootstrap_contacts:
|
||||
log.warning("no bootstrap contacts to ping")
|
||||
ping_result = yield _ping_contacts(bootstrap_contacts)
|
||||
shortlist = ping_result.keys()
|
||||
shortlist = list(ping_result.keys())
|
||||
if not shortlist:
|
||||
log.warning("failed to ping %i bootstrap contacts", len(bootstrap_contacts))
|
||||
defer.returnValue(None)
|
||||
@@ -228,10 +216,10 @@ class Node(MockKademliaHelper):
|
||||
# find the closest peers to us
|
||||
closest = yield self._iterativeFind(self.node_id, shortlist if not self.contacts else None)
|
||||
yield _ping_contacts(closest)
|
||||
# # query random hashes in our bucket key ranges to fill or split them
|
||||
# random_ids_in_range = self._routingTable.getRefreshList()
|
||||
# while random_ids_in_range:
|
||||
# yield self.iterativeFindNode(random_ids_in_range.pop())
|
||||
# query random hashes in our bucket key ranges to fill or split them
|
||||
random_ids_in_range = self._routingTable.getRefreshList()
|
||||
while random_ids_in_range:
|
||||
yield self.iterativeFindNode(random_ids_in_range.pop())
|
||||
defer.returnValue(None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -255,7 +243,7 @@ class Node(MockKademliaHelper):
|
||||
yield _iterative_join()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self, known_node_addresses=None):
|
||||
def start(self, known_node_addresses=None, block_on_join=False):
|
||||
""" Causes the Node to attempt to join the DHT network by contacting the
|
||||
known DHT nodes. This can be called multiple times if the previous attempt
|
||||
has failed or if the Node has lost all the contacts.
|
||||
@@ -270,8 +258,11 @@ class Node(MockKademliaHelper):
|
||||
self.start_listening()
|
||||
yield self._protocol._listening
|
||||
# TODO: Refresh all k-buckets further away than this node's closest neighbour
|
||||
yield self.joinNetwork(known_node_addresses or [])
|
||||
self.start_looping_calls()
|
||||
d = self.joinNetwork(known_node_addresses or [])
|
||||
d.addCallback(lambda _: self.start_looping_calls())
|
||||
d.addCallback(lambda _: log.info("Joined the dht"))
|
||||
if block_on_join:
|
||||
yield d
|
||||
|
||||
def start_looping_calls(self):
|
||||
self.safe_start_looping_call(self._change_token_lc, constants.tokenSecretChangeInterval)
|
||||
@@ -299,25 +290,24 @@ class Node(MockKademliaHelper):
|
||||
@defer.inlineCallbacks
|
||||
def storeToContact(self, blob_hash, contact):
|
||||
try:
|
||||
token = contact.token
|
||||
if not token:
|
||||
find_value_response = yield contact.findValue(blob_hash)
|
||||
token = find_value_response[b'token']
|
||||
contact.update_token(token)
|
||||
res = yield contact.store(blob_hash, token, self.peerPort, self.node_id, 0)
|
||||
if not contact.token:
|
||||
yield contact.findValue(blob_hash)
|
||||
res = yield contact.store(blob_hash, contact.token, self.peerPort, self.node_id, 0)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
defer.returnValue(True)
|
||||
log.debug("Stored %s to %s (%s)", binascii.hexlify(blob_hash), contact.log_id(), contact.address)
|
||||
return True
|
||||
except protocol.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s",
|
||||
binascii.hexlify(blob_hash), contact.log_id())
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
contact.update_token(None)
|
||||
log.error("Unexpected error while storing blob_hash %s at %s: %s",
|
||||
binascii.hexlify(blob_hash), contact, err)
|
||||
defer.returnValue(False)
|
||||
return False
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def announceHaveBlob(self, blob_hash):
|
||||
@@ -451,20 +441,10 @@ class Node(MockKademliaHelper):
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
try:
|
||||
contact = self._routingTable.getContact(contactID)
|
||||
df = defer.Deferred()
|
||||
df.callback(contact)
|
||||
df = defer.succeed(self._routingTable.getContact(contactID))
|
||||
except (ValueError, IndexError):
|
||||
def parseResults(nodes):
|
||||
node_ids = [c.id for c in nodes]
|
||||
if contactID in nodes:
|
||||
contact = nodes[node_ids.index(contactID)]
|
||||
return contact
|
||||
else:
|
||||
return None
|
||||
|
||||
df = self.iterativeFindNode(contactID)
|
||||
df.addCallback(parseResults)
|
||||
df.addCallback(lambda nodes: ([node for node in nodes if node.id == contactID] or (None,))[0])
|
||||
return df
|
||||
|
||||
@rpcmethod
|
||||
@@ -510,7 +490,7 @@ class Node(MockKademliaHelper):
|
||||
if 0 <= port <= 65536:
|
||||
compact_port = port.to_bytes(2, 'big')
|
||||
else:
|
||||
raise TypeError('Invalid port: {}'.format(port))
|
||||
raise TypeError(f'Invalid port: {port}')
|
||||
compact_address = compact_ip + compact_port + rpc_contact.id
|
||||
now = int(self.clock.seconds())
|
||||
originallyPublished = now - age
|
||||
@@ -591,7 +571,7 @@ class Node(MockKademliaHelper):
|
||||
"""
|
||||
return generate_id()
|
||||
|
||||
# from lbrynet.core.utils import profile_deferred
|
||||
# from lbrynet.p2p.utils import profile_deferred
|
||||
# @profile_deferred()
|
||||
@defer.inlineCallbacks
|
||||
def _iterativeFind(self, key, startupShortlist=None, rpc='findNode', exclude=None):
|
||||
@@ -632,7 +612,7 @@ class Node(MockKademliaHelper):
|
||||
# # Update the "last accessed" timestamp for the appropriate k-bucket
|
||||
# self._routingTable.touchKBucket(key)
|
||||
if len(shortlist) == 0:
|
||||
log.warning("This node doesnt know any other nodes")
|
||||
log.warning("This node doesn't know any other nodes")
|
||||
# This node doesn't know of any other nodes
|
||||
fakeDf = defer.Deferred()
|
||||
fakeDf.callback([])
|
||||
@@ -651,19 +631,14 @@ class Node(MockKademliaHelper):
|
||||
replication/republishing as necessary """
|
||||
yield self._refreshRoutingTable()
|
||||
self._dataStore.removeExpiredPeers()
|
||||
yield self._refreshStoringPeers()
|
||||
self._refreshStoringPeers()
|
||||
defer.returnValue(None)
|
||||
|
||||
def _refreshContacts(self):
|
||||
return defer.DeferredList(
|
||||
[self._protocol._ping_queue.enqueue_maybe_ping(contact, delay=0) for contact in self.contacts]
|
||||
)
|
||||
self._protocol._ping_queue.enqueue_maybe_ping(*self.contacts, delay=0)
|
||||
|
||||
def _refreshStoringPeers(self):
|
||||
storing_contacts = self._dataStore.getStoringContacts()
|
||||
return defer.DeferredList(
|
||||
[self._protocol._ping_queue.enqueue_maybe_ping(contact, delay=0) for contact in storing_contacts]
|
||||
)
|
||||
self._protocol._ping_queue.enqueue_maybe_ping(*self._dataStore.getStoringContacts(), delay=0)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _refreshRoutingTable(self):
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
import binascii
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
from lbrynet import conf
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DummyPeerFinder:
|
||||
"""This class finds peers which have announced to the DHT that they have certain blobs"""
|
||||
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
|
||||
return defer.succeed([])
|
||||
|
||||
|
||||
class DHTPeerFinder(DummyPeerFinder):
|
||||
"""This class finds peers which have announced to the DHT that they have certain blobs"""
|
||||
#implements(IPeerFinder)
|
||||
|
||||
def __init__(self, dht_node, peer_manager):
|
||||
"""
|
||||
dht_node - an instance of dht.Node class
|
||||
peer_manager - an instance of PeerManager class
|
||||
"""
|
||||
self.dht_node = dht_node
|
||||
self.peer_manager = peer_manager
|
||||
self.peers = {}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
|
||||
"""
|
||||
Find peers for blob in the DHT
|
||||
blob_hash (str): blob hash to look for
|
||||
timeout (int): seconds to timeout after
|
||||
filter_self (bool): if True, and if a peer for a blob is itself, filter it
|
||||
from the result
|
||||
|
||||
Returns:
|
||||
list of peers for the blob
|
||||
"""
|
||||
if blob_hash not in self.peers:
|
||||
self.peers[blob_hash] = [(self.dht_node.externalIP, self.dht_node.peerPort)]
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
finished_deferred = self.dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
|
||||
timeout = timeout or conf.settings['peer_search_timeout']
|
||||
if timeout:
|
||||
finished_deferred.addTimeout(timeout, self.dht_node.clock)
|
||||
try:
|
||||
peer_list = yield finished_deferred
|
||||
except defer.TimeoutError:
|
||||
log.debug("DHT timed out while looking peers for blob %s after %s seconds",
|
||||
blob_hash, timeout)
|
||||
peer_list = []
|
||||
|
||||
peers = set(peer_list)
|
||||
results = []
|
||||
for node_id, host, port in peers:
|
||||
if filter_self and (host, port) == (self.dht_node.externalIP, self.dht_node.peerPort):
|
||||
continue
|
||||
results.append(self.peer_manager.get_peer(host, port))
|
||||
self.peers[blob_hash].append((host, port))
|
||||
defer.returnValue(results)
|
||||
@@ -1,15 +1,10 @@
|
||||
import logging
|
||||
import errno
|
||||
from binascii import hexlify
|
||||
from collections import deque
|
||||
|
||||
from twisted.internet import protocol, defer
|
||||
from .error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
|
||||
|
||||
from . import constants
|
||||
from . import encoding
|
||||
from . import msgtypes
|
||||
from . import msgformat
|
||||
from lbrynet.dht import constants, encoding, msgformat, msgtypes
|
||||
from lbrynet.dht.error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -22,73 +17,53 @@ class PingQueue:
|
||||
|
||||
def __init__(self, node):
|
||||
self._node = node
|
||||
self._get_time = self._node.clock.seconds
|
||||
self._queue = deque()
|
||||
self._enqueued_contacts = {}
|
||||
self._semaphore = defer.DeferredSemaphore(1)
|
||||
self._ping_semaphore = defer.DeferredSemaphore(constants.alpha)
|
||||
self._process_lc = node.get_looping_call(self._semaphore.run, self._process)
|
||||
self._pending_contacts = {}
|
||||
self._process_lc = node.get_looping_call(self._process)
|
||||
|
||||
def _add_contact(self, contact, delay=None):
|
||||
if (contact.address, contact.port) in [(c.address, c.port) for c in self._enqueued_contacts]:
|
||||
return defer.succeed(None)
|
||||
delay = delay or constants.checkRefreshInterval
|
||||
self._enqueued_contacts[contact] = self._get_time() + delay
|
||||
self._queue.append(contact)
|
||||
return defer.succeed(None)
|
||||
def enqueue_maybe_ping(self, *contacts, **kwargs):
|
||||
delay = kwargs.get('delay', constants.checkRefreshInterval)
|
||||
no_op = (defer.succeed(None), lambda: None)
|
||||
for contact in contacts:
|
||||
if delay and contact not in self._enqueued_contacts:
|
||||
self._pending_contacts.setdefault(contact, self._node.clock.seconds() + delay)
|
||||
else:
|
||||
self._enqueued_contacts.setdefault(contact, no_op)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _process(self):
|
||||
if not len(self._queue):
|
||||
defer.returnValue(None)
|
||||
contact = self._queue.popleft()
|
||||
now = self._get_time()
|
||||
|
||||
# if the oldest contact in the queue isn't old enough to be pinged, add it back to the queue and return
|
||||
if now < self._enqueued_contacts[contact]:
|
||||
self._queue.appendleft(contact)
|
||||
defer.returnValue(None)
|
||||
|
||||
pinged = []
|
||||
checked = []
|
||||
while now > self._enqueued_contacts[contact]:
|
||||
checked.append(contact)
|
||||
if not contact.contact_is_good:
|
||||
pinged.append(contact)
|
||||
if not len(self._queue):
|
||||
break
|
||||
contact = self._queue.popleft()
|
||||
if not now > self._enqueued_contacts[contact]:
|
||||
checked.append(contact)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _ping(contact):
|
||||
try:
|
||||
yield contact.ping()
|
||||
except TimeoutError:
|
||||
pass
|
||||
except Exception as err:
|
||||
log.warning("unexpected error: %s", err)
|
||||
|
||||
yield defer.DeferredList([_ping(contact) for contact in pinged])
|
||||
|
||||
for contact in checked:
|
||||
if contact in self._enqueued_contacts and contact in pinged:
|
||||
def _ping(self, contact):
|
||||
if contact.contact_is_good:
|
||||
return
|
||||
try:
|
||||
yield contact.ping()
|
||||
except TimeoutError:
|
||||
pass
|
||||
except Exception as err:
|
||||
log.warning("unexpected error: %s", err)
|
||||
finally:
|
||||
if contact in self._enqueued_contacts:
|
||||
del self._enqueued_contacts[contact]
|
||||
elif contact not in self._queue:
|
||||
self._queue.appendleft(contact)
|
||||
|
||||
defer.returnValue(None)
|
||||
def _process(self):
|
||||
# move contacts that are scheduled to join the queue
|
||||
if self._pending_contacts:
|
||||
now = self._node.clock.seconds()
|
||||
for contact in [contact for contact, schedule in self._pending_contacts.items() if schedule <= now]:
|
||||
del self._pending_contacts[contact]
|
||||
self._enqueued_contacts.setdefault(contact, (defer.succeed(None), lambda: None))
|
||||
# spread pings across 60 seconds to avoid flood and/or false negatives
|
||||
step = 60.0/float(len(self._enqueued_contacts)) if self._enqueued_contacts else 0
|
||||
for index, (contact, (call, _)) in enumerate(self._enqueued_contacts.items()):
|
||||
if call.called and not contact.contact_is_good:
|
||||
self._enqueued_contacts[contact] = self._node.reactor_callLater(index*step, self._ping, contact)
|
||||
|
||||
def start(self):
|
||||
return self._node.safe_start_looping_call(self._process_lc, 60)
|
||||
|
||||
def stop(self):
|
||||
map(None, (cancel() for _, (call, cancel) in self._enqueued_contacts.items() if not call.called))
|
||||
return self._node.safe_stop_looping_call(self._process_lc)
|
||||
|
||||
def enqueue_maybe_ping(self, contact, delay=None):
|
||||
return self._semaphore.run(self._add_contact, contact, delay)
|
||||
|
||||
|
||||
class KademliaProtocol(protocol.DatagramProtocol):
|
||||
""" Implements all low-level network-related functions of a Kademlia node """
|
||||
@@ -139,7 +114,20 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
return args
|
||||
return args + ({b'protocolVersion': self._protocolVersion},)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def sendRPC(self, contact, method, args):
|
||||
for _ in range(constants.rpcAttempts):
|
||||
try:
|
||||
response = yield self._sendRPC(contact, method, args)
|
||||
return response
|
||||
except TimeoutError:
|
||||
if contact.contact_is_good:
|
||||
log.debug("RETRY %s ON %s", method, contact)
|
||||
continue
|
||||
else:
|
||||
raise
|
||||
|
||||
def _sendRPC(self, contact, method, args):
|
||||
"""
|
||||
Sends an RPC to the specified contact
|
||||
|
||||
@@ -174,16 +162,19 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
df = defer.Deferred()
|
||||
|
||||
def _remove_contact(failure): # remove the contact from the routing table and track the failure
|
||||
contact.update_last_failed()
|
||||
try:
|
||||
self._node.removeContact(contact)
|
||||
if not contact.contact_is_good:
|
||||
self._node.removeContact(contact)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
contact.update_last_failed()
|
||||
return failure
|
||||
|
||||
def _update_contact(result): # refresh the contact in the routing table
|
||||
contact.update_last_replied()
|
||||
if method == b'findValue':
|
||||
if b'token' in result:
|
||||
contact.update_token(result[b'token'])
|
||||
if b'protocolVersion' not in result:
|
||||
contact.update_protocol_version(0)
|
||||
else:
|
||||
@@ -240,7 +231,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
message = self._translator.fromPrimitive(msgPrimitive)
|
||||
except (encoding.DecodeError, ValueError) as err:
|
||||
# We received some rubbish here
|
||||
log.warning("Error decoding datagram %s from %s:%i - %s", datagram.encode('hex'),
|
||||
log.warning("Error decoding datagram %s from %s:%i - %s", hexlify(datagram),
|
||||
address[0], address[1], err)
|
||||
return
|
||||
except (IndexError, KeyError):
|
||||
@@ -330,7 +321,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
If the data is spread over multiple UDP datagrams, the packets have the
|
||||
following structure::
|
||||
| | | | | |||||||||||| 0x00 |
|
||||
|Transmision|Total number|Sequence number| RPC ID |Header end|
|
||||
|Transmission|Total number|Sequence number| RPC ID |Header end|
|
||||
| type ID | of packets |of this packet | | indicator|
|
||||
| (1 byte) | (2 bytes) | (2 bytes) |(20 bytes)| (1 byte) |
|
||||
| | | | | |||||||||||| |
|
||||
@@ -357,7 +348,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
while seqNumber < totalPackets:
|
||||
packetData = data[startPos:startPos + self.msgSizeLimit]
|
||||
encSeqNumber = chr(seqNumber >> 8) + chr(seqNumber & 0xff)
|
||||
txData = '\x00%s%s%s\x00%s' % (encTotalPackets, encSeqNumber, rpcID, packetData)
|
||||
txData = f'\x00{encTotalPackets}{encSeqNumber}{rpcID}\x00{packetData}'
|
||||
self._scheduleSendNext(txData, address)
|
||||
|
||||
startPos += self.msgSizeLimit
|
||||
@@ -376,7 +367,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
except OSError as err:
|
||||
if err.errno == errno.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesnt seem to affect anything -grin
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
elif err.errno == errno.ENETUNREACH:
|
||||
# this should probably try to retransmit when the network connection is back
|
||||
@@ -441,7 +432,7 @@ class KademliaProtocol(protocol.DatagramProtocol):
|
||||
else:
|
||||
result = func()
|
||||
except Exception as e:
|
||||
log.exception("error handling request for %s:%i %s", senderContact.address, senderContact.port, method)
|
||||
log.error("error handling request for %s:%i %s", senderContact.address, senderContact.port, method)
|
||||
df.errback(e)
|
||||
else:
|
||||
df.callback(result)
|
||||
|
||||
@@ -6,14 +6,12 @@
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
import random
|
||||
from binascii import unhexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from . import constants
|
||||
from . import kbucket
|
||||
from .error import TimeoutError
|
||||
from .distance import Distance
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.dht import constants, kbucket
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht.distance import Distance
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -48,6 +46,7 @@ class TreeRoutingTable:
|
||||
from twisted.internet import reactor
|
||||
getTime = reactor.seconds
|
||||
self._getTime = getTime
|
||||
self._ongoing_replacements = set()
|
||||
|
||||
def get_contacts(self):
|
||||
contacts = []
|
||||
@@ -134,8 +133,14 @@ class TreeRoutingTable:
|
||||
to_replace = not_good_contacts[0]
|
||||
else:
|
||||
to_replace = self._buckets[bucketIndex]._contacts[0]
|
||||
df = to_replace.ping()
|
||||
df.addErrback(replaceContact, to_replace)
|
||||
if to_replace not in self._ongoing_replacements:
|
||||
log.debug("pinging %s:%s", to_replace.address, to_replace.port)
|
||||
self._ongoing_replacements.add(to_replace)
|
||||
df = to_replace.ping()
|
||||
df.addErrback(replaceContact, to_replace)
|
||||
df.addBoth(lambda _: self._ongoing_replacements.remove(to_replace))
|
||||
else:
|
||||
df = defer.succeed(None)
|
||||
return df
|
||||
else:
|
||||
self.touchKBucketByIndex(bucketIndex)
|
||||
@@ -150,7 +155,7 @@ class TreeRoutingTable:
|
||||
@param count: the amount of contacts to return, default of k (8)
|
||||
@type count: int
|
||||
@param sender_node_id: Used during RPC, this is be the sender's Node ID
|
||||
Whatever ID is passed in the paramater will get
|
||||
Whatever ID is passed in the parameter will get
|
||||
excluded from the list of returned contacts.
|
||||
@type sender_node_id: str
|
||||
|
||||
@@ -208,7 +213,7 @@ class TreeRoutingTable:
|
||||
now = int(self._getTime())
|
||||
for bucket in self._buckets[startIndex:]:
|
||||
if force or now - bucket.lastAccessed >= constants.refreshTimeout:
|
||||
searchID = self._randomIDInBucketRange(bucketIndex)
|
||||
searchID = self.midpoint_id_in_bucket_range(bucketIndex)
|
||||
refreshIDs.append(searchID)
|
||||
bucketIndex += 1
|
||||
return refreshIDs
|
||||
@@ -256,22 +261,25 @@ class TreeRoutingTable:
|
||||
i += 1
|
||||
return i
|
||||
|
||||
def _randomIDInBucketRange(self, bucketIndex):
|
||||
def random_id_in_bucket_range(self, bucketIndex):
|
||||
""" Returns a random ID in the specified k-bucket's range
|
||||
|
||||
@param bucketIndex: The index of the k-bucket to use
|
||||
@type bucketIndex: int
|
||||
"""
|
||||
idValue = random.randrange(
|
||||
self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax)
|
||||
randomID = hex(idValue)[2:]
|
||||
if randomID[-1] == 'L':
|
||||
randomID = randomID[:-1]
|
||||
if len(randomID) % 2 != 0:
|
||||
randomID = '0' + randomID
|
||||
randomID = unhexlify(randomID)
|
||||
randomID = ((constants.key_bits // 8) - len(randomID)) * b'\x00' + randomID
|
||||
return randomID
|
||||
|
||||
random_id = int(random.randrange(self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax))
|
||||
return random_id.to_bytes(constants.key_bits // 8, 'big')
|
||||
|
||||
def midpoint_id_in_bucket_range(self, bucketIndex):
|
||||
""" Returns the middle ID in the specified k-bucket's range
|
||||
|
||||
@param bucketIndex: The index of the k-bucket to use
|
||||
@type bucketIndex: int
|
||||
"""
|
||||
|
||||
half = int((self._buckets[bucketIndex].rangeMax - self._buckets[bucketIndex].rangeMin) // 2)
|
||||
return int(self._buckets[bucketIndex].rangeMin + half).to_bytes(constants.key_bits // 8, 'big')
|
||||
|
||||
def _splitBucket(self, oldBucketIndex):
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
import sys
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
import logging
|
||||
from docopt import docopt
|
||||
from textwrap import dedent
|
||||
|
||||
import aiohttp
|
||||
from lbrynet.extras.compat import force_asyncioreactor_install
|
||||
force_asyncioreactor_install()
|
||||
|
||||
from lbrynet import log_support, __name__ as lbrynet_name, __version__ as lbrynet_version
|
||||
from lbrynet.extras.daemon.loggly_handler import get_loggly_handler
|
||||
from lbrynet.conf import Config, CLIConfig
|
||||
from lbrynet.utils import check_connection
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
|
||||
log = logging.getLogger(lbrynet_name)
|
||||
log.addHandler(logging.NullHandler())
|
||||
|
||||
|
||||
def display(data):
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
async def execute_command(conf, method, params):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
message = {'method': method, 'params': params}
|
||||
async with session.get(conf.api_connection_url, json=message) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
display(data['result'])
|
||||
elif 'error' in data:
|
||||
if 'message' in data['error']:
|
||||
display(data['error']['message'])
|
||||
else:
|
||||
display(data['error'])
|
||||
except Exception as e:
|
||||
log.exception('Could not process response from server:', exc_info=e)
|
||||
except aiohttp.ClientConnectionError:
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
|
||||
|
||||
def print_help():
|
||||
print(dedent("""
|
||||
NAME
|
||||
lbrynet - LBRY command line client.
|
||||
|
||||
USAGE
|
||||
lbrynet [--data_dir=<blob and database directory>] [--wallet_dir=<wallet directory>]
|
||||
[--download_dir=<downloads directory>] <command> [<args>]
|
||||
|
||||
EXAMPLES
|
||||
lbrynet start # starts the daemon and listens for jsonrpc commands
|
||||
lbrynet help # display this message
|
||||
lbrynet help <command_name> # get help for a command(doesn't need the daemon to be running)
|
||||
lbrynet commands # list available commands
|
||||
lbrynet status # get the running status of the daemon
|
||||
lbrynet resolve what # resolve a name
|
||||
|
||||
lbrynet --wallet_dir=~/wallet2 start # start the daemon using an alternative wallet directory
|
||||
lbrynet --data_dir=~/lbry start # start the daemon using an alternative data directory
|
||||
|
||||
lbrynet --data_dir=~/lbry <command_name> # run a command on a daemon using an alternative data directory,
|
||||
# which can contain a full daemon_settings.yml config file.
|
||||
# Note: since the daemon is what runs the wallet and
|
||||
# downloads files, only the --data_dir setting is needed when
|
||||
# running commands. The wallet_dir and download_dir would only
|
||||
# by used when starting the daemon.
|
||||
"""))
|
||||
|
||||
|
||||
def print_help_for_command(command):
|
||||
fn = Daemon.callable_methods.get(command)
|
||||
if fn:
|
||||
print(dedent(fn.__doc__))
|
||||
else:
|
||||
print("Invalid command name")
|
||||
|
||||
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
|
||||
return x
|
||||
if x.lower() == 'true':
|
||||
return True
|
||||
if x.lower() == 'false':
|
||||
return False
|
||||
if x.isdigit():
|
||||
return int(x)
|
||||
return x
|
||||
|
||||
|
||||
def remove_brackets(key):
|
||||
if key.startswith("<") and key.endswith(">"):
|
||||
return str(key[1:-1])
|
||||
return key
|
||||
|
||||
|
||||
def set_kwargs(parsed_args):
|
||||
kwargs = {}
|
||||
for key, arg in parsed_args.items():
|
||||
k = None
|
||||
if arg is None:
|
||||
continue
|
||||
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
kwargs[k] = normalize_value(arg, k)
|
||||
return kwargs
|
||||
|
||||
|
||||
def get_argument_parser():
|
||||
main = argparse.ArgumentParser('lbrynet', add_help=False)
|
||||
main.add_argument(
|
||||
'--version', dest='cli_version', action="store_true",
|
||||
help='Show lbrynet CLI version and exit.'
|
||||
)
|
||||
main.add_argument(
|
||||
'-h', '--help', dest='help', action="store_true",
|
||||
help='Show this help message and exit'
|
||||
)
|
||||
CLIConfig.contribute_args(main)
|
||||
sub = main.add_subparsers(dest='command')
|
||||
help = sub.add_parser('help', help='Detailed help for remote commands.')
|
||||
help.add_argument('help_command', nargs='*')
|
||||
start = sub.add_parser('start', help='Start lbrynet server.')
|
||||
start.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
Config.contribute_args(start)
|
||||
api = Daemon.get_api_definitions()
|
||||
for group in sorted(api):
|
||||
group_command = sub.add_parser(group, help=api[group]['doc'])
|
||||
group_command.set_defaults(group_doc=group_command)
|
||||
if group in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
|
||||
continue
|
||||
commands = group_command.add_subparsers(dest='subcommand')
|
||||
for command in api[group]['commands']:
|
||||
commands.add_parser(command['name'], help=command['doc'].strip().splitlines()[0])
|
||||
for deprecated in Daemon.deprecated_methods:
|
||||
group_command = sub.add_parser(deprecated)
|
||||
group_command.add_subparsers(dest='subcommand')
|
||||
return main
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
conf = Config.create_from_arguments(args)
|
||||
|
||||
if args.cli_version:
|
||||
print(f"{lbrynet_name} {lbrynet_version}")
|
||||
return 0
|
||||
|
||||
elif args.command == 'start':
|
||||
|
||||
log_support.configure_logging(conf.log_file_path, not args.quiet, args.verbose)
|
||||
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
log.addHandler(loggly_handler)
|
||||
|
||||
log.debug('Final Settings: %s', conf.settings_dict)
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
daemon = Daemon(conf)
|
||||
|
||||
if check_connection():
|
||||
from twisted.internet import reactor
|
||||
reactor._asyncioEventloop.create_task(daemon.start())
|
||||
reactor.run()
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
|
||||
elif args.command == 'help':
|
||||
|
||||
if args.help_command:
|
||||
method = '_'.join(args.help_command)
|
||||
else:
|
||||
parser.print_help()
|
||||
return 0
|
||||
|
||||
if method not in Daemon.callable_methods:
|
||||
print('Invalid command name: {method}')
|
||||
return 1
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
print(fn.__doc__)
|
||||
|
||||
elif args.command is not None:
|
||||
|
||||
if args.command in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
|
||||
method = args.command
|
||||
elif args.subcommand is not None:
|
||||
method = f'{args.command}_{args.subcommand}'
|
||||
else:
|
||||
args.group_doc.print_help()
|
||||
return 0
|
||||
|
||||
if method in Daemon.deprecated_methods:
|
||||
new_method = Daemon.deprecated_methods[method].new_command
|
||||
if new_method is None:
|
||||
print(f"{method} is permanently deprecated and does not have a replacement command.")
|
||||
return 0
|
||||
print(f"{method} is deprecated, using {new_method}.")
|
||||
method = new_method
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
parsed = docopt(fn.__doc__, command_args)
|
||||
params = set_kwargs(parsed)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(execute_command(conf, method, params))
|
||||
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,25 @@
|
||||
import asyncio
|
||||
|
||||
|
||||
def force_asyncioreactor_install():
|
||||
import sys
|
||||
from twisted.internet import asyncioreactor
|
||||
if 'twisted.internet.reactor' not in sys.modules:
|
||||
asyncioreactor.install()
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
|
||||
# pyinstaller hooks install the default reactor before
|
||||
# any of our code runs, see kivy for similar problem:
|
||||
# https://github.com/kivy/kivy/issues/4182
|
||||
del sys.modules['twisted.internet.reactor']
|
||||
asyncioreactor.install()
|
||||
|
||||
|
||||
def d2f(deferred):
|
||||
return deferred.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def f2d(future):
|
||||
from twisted.internet import defer
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
@@ -1,7 +1,9 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted._threads import AlreadyQuit
|
||||
from .ComponentManager import ComponentManager
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -26,6 +28,7 @@ class Component(metaclass=ComponentType):
|
||||
component_name = None
|
||||
|
||||
def __init__(self, component_manager):
|
||||
self.conf: Config = component_manager.conf
|
||||
self.component_manager = component_manager
|
||||
self._running = False
|
||||
|
||||
@@ -36,37 +39,35 @@ class Component(metaclass=ComponentType):
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup(self):
|
||||
async def _setup(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.start)
|
||||
result = await self.start()
|
||||
self._running = True
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||
log.exception("Error setting up %s", self.component_name or self.__class__.__name__)
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop(self):
|
||||
async def _stop(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.stop)
|
||||
result = await self.stop()
|
||||
self._running = False
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||
@@ -1,7 +1,9 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.core.Error import ComponentStartConditionNotMet
|
||||
from lbrynet.p2p.Error import ComponentStartConditionNotMet
|
||||
from lbrynet.extras.daemon.PeerManager import PeerManager
|
||||
from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
|
||||
from lbrynet.conf import Config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -33,13 +35,16 @@ class RequiredCondition(metaclass=RequiredConditionType):
|
||||
class ComponentManager:
|
||||
default_component_classes = {}
|
||||
|
||||
def __init__(self, reactor=None, analytics_manager=None, skip_components=None, **override_components):
|
||||
def __init__(self, conf: Config, reactor=None, analytics_manager=None, skip_components=None,
|
||||
peer_manager=None, peer_finder=None, **override_components):
|
||||
self.conf = conf
|
||||
self.skip_components = skip_components or []
|
||||
|
||||
self.reactor = reactor
|
||||
self.component_classes = {}
|
||||
self.components = set()
|
||||
self.analytics_manager = analytics_manager
|
||||
self.peer_manager = peer_manager or PeerManager()
|
||||
self.peer_finder = peer_finder or DHTPeerFinder(self)
|
||||
|
||||
for component_name, component_class in self.default_component_classes.items():
|
||||
if component_name in override_components:
|
||||
@@ -52,18 +57,19 @@ class ComponentManager:
|
||||
|
||||
for component_class in self.component_classes.values():
|
||||
self.components.add(component_class(self))
|
||||
self.daemon = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def evaluate_condition(self, condition_name):
|
||||
if condition_name not in RegisteredConditions.conditions:
|
||||
raise NameError(condition_name)
|
||||
condition = RegisteredConditions.conditions[condition_name]
|
||||
try:
|
||||
component = self.get_component(condition.component)
|
||||
result = yield defer.maybeDeferred(condition.evaluate, component)
|
||||
except Exception as err:
|
||||
result = condition.evaluate(component)
|
||||
except Exception:
|
||||
log.exception('failed to evaluate condition:')
|
||||
result = False
|
||||
defer.returnValue((result, "" if result else condition.message))
|
||||
return result, "" if result else condition.message
|
||||
|
||||
def sort_components(self, reverse=False):
|
||||
"""
|
||||
@@ -107,33 +113,31 @@ class ComponentManager:
|
||||
steps.reverse()
|
||||
return steps
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self, **callbacks):
|
||||
"""
|
||||
Start Components in sequence sorted by requirements
|
||||
|
||||
:return: (defer.Deferred)
|
||||
"""
|
||||
|
||||
async def setup(self, **callbacks):
|
||||
""" Start Components in sequence sorted by requirements """
|
||||
for component_name, cb in callbacks.items():
|
||||
if component_name not in self.component_classes:
|
||||
raise NameError("unknown component: %s" % component_name)
|
||||
if component_name not in self.skip_components:
|
||||
raise NameError("unknown component: %s" % component_name)
|
||||
if not callable(cb):
|
||||
raise ValueError("%s is not callable" % cb)
|
||||
|
||||
def _setup(component):
|
||||
async def _setup(component):
|
||||
await component._setup()
|
||||
if component.component_name in callbacks:
|
||||
d = component._setup()
|
||||
d.addCallback(callbacks[component.component_name], component)
|
||||
return d
|
||||
return component._setup()
|
||||
maybe_coro = callbacks[component.component_name](component)
|
||||
if asyncio.iscoroutine(maybe_coro):
|
||||
asyncio.create_task(maybe_coro)
|
||||
|
||||
stages = self.sort_components()
|
||||
for stage in stages:
|
||||
yield defer.DeferredList([_setup(component) for component in stage if not component.running])
|
||||
needing_start = [
|
||||
_setup(component) for component in stage if not component.running
|
||||
]
|
||||
if needing_start:
|
||||
await asyncio.wait(needing_start)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
"""
|
||||
Stop Components in reversed startup order
|
||||
|
||||
@@ -141,7 +145,11 @@ class ComponentManager:
|
||||
"""
|
||||
stages = self.sort_components(reverse=True)
|
||||
for stage in stages:
|
||||
yield defer.DeferredList([component._stop() for component in stage if component.running])
|
||||
needing_stop = [
|
||||
component._stop() for component in stage if component.running
|
||||
]
|
||||
if needing_stop:
|
||||
await asyncio.wait(needing_stop)
|
||||
|
||||
def all_components_running(self, *component_names):
|
||||
"""
|
||||
@@ -1,33 +1,41 @@
|
||||
import os
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import logging
|
||||
import treq
|
||||
import math
|
||||
import binascii
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
from twisted.internet import defer, threads, reactor, error
|
||||
import lbryschema
|
||||
from txupnp.upnp import UPnP
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.utils import DeferredDict
|
||||
from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager
|
||||
from lbrynet.core.RateLimiter import RateLimiter
|
||||
from lbrynet.core.BlobManager import DiskBlobManager
|
||||
from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
|
||||
from lbrynet.wallet.manager import LbryWalletManager
|
||||
from lbrynet.wallet.network import Network
|
||||
from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory
|
||||
from lbrynet.core.server.ServerProtocol import ServerProtocolFactory
|
||||
from lbrynet.daemon.Component import Component
|
||||
from lbrynet.daemon.ExchangeRateManager import ExchangeRateManager
|
||||
from lbrynet.database.storage import SQLiteStorage
|
||||
from lbrynet.dht import node, hashannouncer
|
||||
from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
|
||||
from lbrynet.lbry_file.client.EncryptedFileDownloader import EncryptedFileSaverFactory
|
||||
from lbrynet.lbry_file.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
|
||||
from lbrynet.reflector import ServerFactory as reflector_server_factory
|
||||
from twisted.internet import defer, reactor, error
|
||||
|
||||
from aioupnp import __version__ as aioupnp_version
|
||||
from aioupnp.upnp import UPnP
|
||||
from aioupnp.fault import UPnPError
|
||||
|
||||
import lbrynet.schema
|
||||
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.extras.compat import d2f
|
||||
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
|
||||
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.Component import Component
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
|
||||
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.wallet import Network
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
|
||||
from lbrynet.p2p.RateLimiter import RateLimiter
|
||||
from lbrynet.p2p.BlobManager import DiskBlobManager
|
||||
from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
|
||||
from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory
|
||||
from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory
|
||||
|
||||
from lbrynet.core.utils import generate_id
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -39,7 +47,6 @@ HEADERS_COMPONENT = "blockchain_headers"
|
||||
WALLET_COMPONENT = "wallet"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
STREAM_IDENTIFIER_COMPONENT = "stream_identifier"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
REFLECTOR_COMPONENT = "reflector"
|
||||
@@ -49,53 +56,23 @@ RATE_LIMITER_COMPONENT = "rate_limiter"
|
||||
PAYMENT_RATE_COMPONENT = "payment_rate_manager"
|
||||
|
||||
|
||||
def get_wallet_config():
|
||||
wallet_type = GCS('wallet')
|
||||
if wallet_type == conf.LBRYCRD_WALLET:
|
||||
raise ValueError('LBRYcrd Wallet is no longer supported')
|
||||
elif wallet_type != conf.LBRYUM_WALLET:
|
||||
raise ValueError('Wallet Type {} is not valid'.format(wallet_type))
|
||||
lbryum_servers = {address: {'t': str(port)}
|
||||
for address, port in GCS('lbryum_servers')}
|
||||
config = {
|
||||
'auto_connect': True,
|
||||
'chain': GCS('blockchain_name'),
|
||||
'default_servers': lbryum_servers
|
||||
}
|
||||
if 'use_keyring' in conf.settings:
|
||||
config['use_keyring'] = GCS('use_keyring')
|
||||
if conf.settings['lbryum_wallet_dir']:
|
||||
config['lbryum_path'] = GCS('lbryum_wallet_dir')
|
||||
return config
|
||||
async def gather_dict(tasks: dict):
|
||||
async def wait_value(key, value):
|
||||
return key, await value
|
||||
return dict(await asyncio.gather(*(
|
||||
wait_value(*kv) for kv in tasks.items()
|
||||
)))
|
||||
|
||||
|
||||
class ConfigSettings:
|
||||
@staticmethod
|
||||
def get_conf_setting(setting_name):
|
||||
return conf.settings[setting_name]
|
||||
|
||||
@staticmethod
|
||||
def get_blobfiles_dir():
|
||||
if conf.settings['BLOBFILES_DIR'] == "blobfiles":
|
||||
return os.path.join(GCS("data_dir"), "blobfiles")
|
||||
else:
|
||||
log.info("Using non-default blobfiles directory: %s", conf.settings['BLOBFILES_DIR'])
|
||||
return conf.settings['BLOBFILES_DIR']
|
||||
|
||||
@staticmethod
|
||||
def get_node_id():
|
||||
return conf.settings.node_id
|
||||
|
||||
@staticmethod
|
||||
def get_external_ip():
|
||||
from lbrynet.core.system_info import get_platform
|
||||
platform = get_platform(get_ip=True)
|
||||
return platform['ip']
|
||||
|
||||
|
||||
# Shorthand for common ConfigSettings methods
|
||||
CS = ConfigSettings
|
||||
GCS = ConfigSettings.get_conf_setting
|
||||
async def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get("https://api.lbry.io/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
@@ -113,38 +90,24 @@ class DatabaseComponent(Component):
|
||||
def get_current_db_revision():
|
||||
return 9
|
||||
|
||||
@staticmethod
|
||||
def get_revision_filename():
|
||||
return conf.settings.get_db_revision_filename()
|
||||
@property
|
||||
def revision_filename(self):
|
||||
return self.component_manager.daemon.db_revision_file_path
|
||||
|
||||
@staticmethod
|
||||
def _write_db_revision_file(version_num):
|
||||
with open(conf.settings.get_db_revision_filename(), mode='w') as db_revision:
|
||||
def _write_db_revision_file(self, version_num):
|
||||
with open(self.revision_filename, mode='w') as db_revision:
|
||||
db_revision.write(str(version_num))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
# check directories exist, create them if they don't
|
||||
log.info("Loading databases")
|
||||
|
||||
if not os.path.exists(GCS('download_directory')):
|
||||
os.mkdir(GCS('download_directory'))
|
||||
|
||||
if not os.path.exists(GCS('data_dir')):
|
||||
os.mkdir(GCS('data_dir'))
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.debug("Created the db revision file: %s", self.get_revision_filename())
|
||||
|
||||
if not os.path.exists(CS.get_blobfiles_dir()):
|
||||
os.mkdir(CS.get_blobfiles_dir())
|
||||
log.debug("Created the blobfile directory: %s", str(CS.get_blobfiles_dir()))
|
||||
|
||||
if not os.path.exists(self.get_revision_filename()):
|
||||
if not os.path.exists(self.revision_filename):
|
||||
log.warning("db_revision file not found. Creating it")
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
|
||||
# check the db migration and run any needed migrations
|
||||
with open(self.get_revision_filename(), "r") as revision_read_handle:
|
||||
with open(self.revision_filename, "r") as revision_read_handle:
|
||||
old_revision = int(revision_read_handle.read().strip())
|
||||
|
||||
if old_revision > self.get_current_db_revision():
|
||||
@@ -152,21 +115,21 @@ class DatabaseComponent(Component):
|
||||
'Your database is revision %i, expected %i' %
|
||||
(old_revision, self.get_current_db_revision()))
|
||||
if old_revision < self.get_current_db_revision():
|
||||
from lbrynet.database.migrator import dbmigrator
|
||||
from lbrynet.extras.daemon.migrator import dbmigrator
|
||||
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
|
||||
yield threads.deferToThread(
|
||||
dbmigrator.migrate_db, GCS('data_dir'), old_revision, self.get_current_db_revision()
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
|
||||
)
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.info("Finished upgrading the databases.")
|
||||
|
||||
# start SQLiteStorage
|
||||
self.storage = SQLiteStorage(GCS('data_dir'))
|
||||
yield self.storage.setup()
|
||||
self.storage = SQLiteStorage(
|
||||
self.conf, os.path.join(self.conf.data_dir, "lbrynet.sqlite")
|
||||
)
|
||||
await self.storage.open()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.storage.stop()
|
||||
async def stop(self):
|
||||
await self.storage.close()
|
||||
self.storage = None
|
||||
|
||||
|
||||
@@ -179,17 +142,17 @@ class HeadersComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.headers_dir = os.path.join(conf.settings['lbryum_wallet_dir'], 'lbc_mainnet')
|
||||
self.headers_dir = os.path.join(self.conf.wallet_dir, 'lbc_mainnet')
|
||||
self.headers_file = os.path.join(self.headers_dir, 'headers')
|
||||
self.old_file = os.path.join(conf.settings['lbryum_wallet_dir'], 'blockchain_headers')
|
||||
self.old_file = os.path.join(self.conf.wallet_dir, 'blockchain_headers')
|
||||
self._downloading_headers = None
|
||||
self._headers_progress_percent = None
|
||||
self._headers_progress_percent = 0
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {} if not self._downloading_headers else {
|
||||
'downloading_headers': self._downloading_headers,
|
||||
'download_progress': self._headers_progress_percent
|
||||
@@ -204,7 +167,7 @@ class HeadersComponent(Component):
|
||||
self._headers_progress_percent = math.ceil(local_size / final_size * 100)
|
||||
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": "bytes={}-".format(local_header_size)}
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
response = yield treq.get(HEADERS_URL, headers=resume_header)
|
||||
got_406 = response.code == 406 # our file is bigger
|
||||
final_size_after_download = response.length + local_header_size
|
||||
@@ -237,41 +200,40 @@ class HeadersComponent(Component):
|
||||
return os.stat(self.headers_file).st_size
|
||||
return 0
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_remote_height(self):
|
||||
async def get_remote_height(self):
|
||||
ledger = SimpleNamespace()
|
||||
ledger.config = {
|
||||
'default_servers': conf.settings['lbryum_servers'],
|
||||
'data_path': conf.settings['lbryum_wallet_dir']
|
||||
'default_servers': self.conf.lbryum_servers,
|
||||
'data_path': self.conf.wallet_dir
|
||||
}
|
||||
net = Network(ledger)
|
||||
net.start()
|
||||
yield net.on_connected.first
|
||||
remote_height = yield net.get_server_height()
|
||||
yield net.stop()
|
||||
defer.returnValue(remote_height)
|
||||
first_connection = net.on_connected.first
|
||||
asyncio.ensure_future(net.start())
|
||||
await first_connection
|
||||
remote_height = await net.get_server_height()
|
||||
await net.stop()
|
||||
return remote_height
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def should_download_headers_from_s3(self):
|
||||
if conf.settings['blockchain_name'] != "lbrycrd_main":
|
||||
defer.returnValue(False)
|
||||
async def should_download_headers_from_s3(self):
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
return False
|
||||
self._check_header_file_integrity()
|
||||
s3_headers_depth = conf.settings['s3_headers_depth']
|
||||
s3_headers_depth = self.conf.s3_headers_depth
|
||||
if not s3_headers_depth:
|
||||
defer.returnValue(False)
|
||||
return False
|
||||
local_height = self.local_header_file_height()
|
||||
remote_height = yield self.get_remote_height()
|
||||
log.info("remote height: %i, local height: %s", remote_height, local_height)
|
||||
remote_height = await self.get_remote_height()
|
||||
log.info("remote height: %i, local height: %i", remote_height, local_height)
|
||||
if remote_height > (local_height + s3_headers_depth):
|
||||
defer.returnValue(True)
|
||||
defer.returnValue(False)
|
||||
return True
|
||||
return False
|
||||
|
||||
def _check_header_file_integrity(self):
|
||||
# TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
|
||||
if conf.settings['blockchain_name'] != "lbrycrd_main":
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
return
|
||||
hashsum = sha256()
|
||||
checksum_height, checksum = conf.settings['HEADERS_FILE_SHA256_CHECKSUM']
|
||||
checksum_height, checksum = HEADERS_FILE_SHA256_CHECKSUM
|
||||
checksum_length_in_bytes = checksum_height * HEADER_SIZE
|
||||
if self.local_header_file_size() < checksum_length_in_bytes:
|
||||
return
|
||||
@@ -279,7 +241,7 @@ class HeadersComponent(Component):
|
||||
hashsum.update(headers_file.read(checksum_length_in_bytes))
|
||||
current_checksum = hashsum.hexdigest()
|
||||
if current_checksum != checksum:
|
||||
msg = "Expected checksum {}, got {}".format(checksum, current_checksum)
|
||||
msg = f"Expected checksum {checksum}, got {current_checksum}"
|
||||
log.warning("Wallet file corrupted, checksum mismatch. " + msg)
|
||||
log.warning("Deleting header file so it can be downloaded again.")
|
||||
os.unlink(self.headers_file)
|
||||
@@ -288,24 +250,23 @@ class HeadersComponent(Component):
|
||||
with open(self.headers_file, "rb+") as headers_file:
|
||||
headers_file.truncate(checksum_length_in_bytes)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
if not os.path.exists(self.headers_dir):
|
||||
os.mkdir(self.headers_dir)
|
||||
if os.path.exists(self.old_file):
|
||||
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
|
||||
os.rename(self.old_file, self.headers_file)
|
||||
self._downloading_headers = yield self.should_download_headers_from_s3()
|
||||
self._downloading_headers = await self.should_download_headers_from_s3()
|
||||
if self._downloading_headers:
|
||||
try:
|
||||
yield self.fetch_headers_from_s3()
|
||||
await d2f(self.fetch_headers_from_s3())
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self._downloading_headers = False
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class WalletComponent(Component):
|
||||
@@ -320,38 +281,35 @@ class WalletComponent(Component):
|
||||
def component(self):
|
||||
return self.wallet_manager
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_status(self):
|
||||
if self.wallet_manager:
|
||||
async def get_status(self):
|
||||
if self.wallet_manager and self.running:
|
||||
local_height = self.wallet_manager.network.get_local_height()
|
||||
remote_height = self.wallet_manager.network.get_server_height()
|
||||
best_hash = yield self.wallet_manager.get_best_blockhash()
|
||||
defer.returnValue({
|
||||
best_hash = self.wallet_manager.get_best_blockhash()
|
||||
return {
|
||||
'blocks': max(local_height, 0),
|
||||
'blocks_behind': max(remote_height - local_height, 0),
|
||||
'best_blockhash': best_hash,
|
||||
'is_encrypted': self.wallet_manager.wallet.use_encryption,
|
||||
'is_locked': not self.wallet_manager.is_wallet_unlocked,
|
||||
})
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
log.info("Starting torba wallet")
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
lbryschema.BLOCKCHAIN_NAME = conf.settings['blockchain_name']
|
||||
self.wallet_manager = LbryWalletManager.from_lbrynet_config(conf.settings, storage)
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = self.conf.blockchain_name
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf, storage)
|
||||
self.wallet_manager.old_db = storage
|
||||
yield self.wallet_manager.start()
|
||||
await self.wallet_manager.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.wallet_manager.stop()
|
||||
async def stop(self):
|
||||
await self.wallet_manager.stop()
|
||||
self.wallet_manager = None
|
||||
|
||||
|
||||
class BlobComponent(Component):
|
||||
component_name = BLOB_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT, DHT_COMPONENT]
|
||||
depends_on = [DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
@@ -363,21 +321,22 @@ class BlobComponent(Component):
|
||||
|
||||
def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, dht_node._dataStore)
|
||||
datastore = None
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_node:
|
||||
datastore = dht_node._dataStore
|
||||
self.blob_manager = DiskBlobManager(os.path.join(self.conf.data_dir, "blobfiles"), storage, datastore)
|
||||
return self.blob_manager.setup()
|
||||
|
||||
def stop(self):
|
||||
return self.blob_manager.stop()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
count = 0
|
||||
if self.blob_manager:
|
||||
count = yield self.blob_manager.storage.count_finished_blobs()
|
||||
defer.returnValue({
|
||||
'finished_blobs': count
|
||||
})
|
||||
count = await self.blob_manager.storage.count_finished_blobs()
|
||||
return {'finished_blobs': count}
|
||||
|
||||
|
||||
class DHTComponent(Component):
|
||||
@@ -395,39 +354,39 @@ class DHTComponent(Component):
|
||||
def component(self):
|
||||
return self.dht_node
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {
|
||||
'node_id': binascii.hexlify(CS.get_node_id()),
|
||||
'node_id': binascii.hexlify(self.component_manager.daemon.node_id),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", GCS("peer_port"))
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", GCS("dht_node_port"))
|
||||
node_id = CS.get_node_id()
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
|
||||
node_id = self.component_manager.daemon.node_id
|
||||
if node_id is None:
|
||||
node_id = generate_id()
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
external_ip = await get_external_ip()
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
self.dht_node = node.Node(
|
||||
self.dht_node = Node(
|
||||
node_id=node_id,
|
||||
udpPort=GCS('dht_node_port'),
|
||||
udpPort=self.conf.dht_node_port,
|
||||
externalUDPPort=self.external_udp_port,
|
||||
externalIP=self.upnp_component.external_ip,
|
||||
externalIP=external_ip,
|
||||
peerPort=self.external_peer_port
|
||||
)
|
||||
|
||||
self.dht_node.start_listening()
|
||||
yield self.dht_node._protocol._listening
|
||||
d = self.dht_node.joinNetwork(GCS('known_dht_nodes'))
|
||||
d.addCallback(lambda _: self.dht_node.start_looping_calls())
|
||||
d.addCallback(lambda _: log.info("Joined the dht"))
|
||||
await d2f(self.dht_node.start(self.conf.known_dht_nodes, block_on_join=False))
|
||||
log.info("Started the dht")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.dht_node.stop()
|
||||
return d2f(self.dht_node.stop())
|
||||
|
||||
|
||||
class HashAnnouncerComponent(Component):
|
||||
@@ -442,18 +401,16 @@ class HashAnnouncerComponent(Component):
|
||||
def component(self):
|
||||
return self.hash_announcer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = hashannouncer.DHTHashAnnouncer(dht_node, storage)
|
||||
yield self.hash_announcer.start()
|
||||
self.hash_announcer = DHTHashAnnouncer(self.conf, dht_node, storage)
|
||||
self.hash_announcer.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.hash_announcer.stop()
|
||||
self.hash_announcer.stop()
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {
|
||||
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.hash_queue)
|
||||
}
|
||||
@@ -470,48 +427,11 @@ class RateLimiterComponent(Component):
|
||||
def component(self):
|
||||
return self.rate_limiter
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
self.rate_limiter.start()
|
||||
return defer.succeed(None)
|
||||
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
self.rate_limiter.stop()
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class StreamIdentifierComponent(Component):
|
||||
component_name = STREAM_IDENTIFIER_COMPONENT
|
||||
depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.sd_identifier = StreamDescriptorIdentifier()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.sd_identifier
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
|
||||
add_lbry_file_to_sd_identifier(self.sd_identifier)
|
||||
file_saver_factory = EncryptedFileSaverFactory(
|
||||
dht_node.peer_finder,
|
||||
rate_limiter,
|
||||
blob_manager,
|
||||
storage,
|
||||
wallet,
|
||||
GCS('download_directory')
|
||||
)
|
||||
yield self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
|
||||
|
||||
def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class PaymentRateComponent(Component):
|
||||
@@ -525,17 +445,17 @@ class PaymentRateComponent(Component):
|
||||
def component(self):
|
||||
return self.payment_rate_manager
|
||||
|
||||
def start(self):
|
||||
return defer.succeed(None)
|
||||
async def start(self):
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
|
||||
STREAM_IDENTIFIER_COMPONENT, PAYMENT_RATE_COMPONENT]
|
||||
depends_on = [RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
|
||||
PAYMENT_RATE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
@@ -545,36 +465,47 @@ class FileManagerComponent(Component):
|
||||
def component(self):
|
||||
return self.file_manager
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
if not self.file_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.file_manager.lbry_files)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
sd_identifier = self.component_manager.get_component(STREAM_IDENTIFIER_COMPONENT)
|
||||
|
||||
sd_identifier = StreamDescriptorIdentifier()
|
||||
add_lbry_file_to_sd_identifier(sd_identifier)
|
||||
file_saver_factory = EncryptedFileSaverFactory(
|
||||
self.conf,
|
||||
self.component_manager.peer_finder,
|
||||
rate_limiter,
|
||||
blob_manager,
|
||||
storage,
|
||||
wallet,
|
||||
self.conf.download_dir
|
||||
)
|
||||
sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
|
||||
|
||||
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
|
||||
log.info('Starting the file manager')
|
||||
self.file_manager = EncryptedFileManager(dht_node.peer_finder, rate_limiter, blob_manager, wallet,
|
||||
payment_rate_manager, storage, sd_identifier)
|
||||
yield self.file_manager.setup()
|
||||
log.info('Done setting up file manager')
|
||||
self.file_manager = EncryptedFileManager(
|
||||
self.conf, self.component_manager.peer_finder, rate_limiter,
|
||||
blob_manager, wallet, payment_rate_manager, storage, sd_identifier
|
||||
)
|
||||
return self.file_manager.setup()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.file_manager.stop()
|
||||
return d2f(self.file_manager.stop())
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
component_name = PEER_PROTOCOL_SERVER_COMPONENT
|
||||
depends_on = [UPNP_COMPONENT, DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
|
||||
depends_on = [UPNP_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
|
||||
PAYMENT_RATE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
@@ -585,11 +516,10 @@ class PeerProtocolServerComponent(Component):
|
||||
def component(self):
|
||||
return self.lbry_server_port
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
peer_port = GCS('peer_port')
|
||||
peer_port = self.conf.peer_port
|
||||
query_handlers = {
|
||||
handler.get_primary_query_identifier(): handler for handler in [
|
||||
BlobRequestHandlerFactory(
|
||||
@@ -603,13 +533,13 @@ class PeerProtocolServerComponent(Component):
|
||||
}
|
||||
server_factory = ServerProtocolFactory(
|
||||
self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers,
|
||||
self.component_manager.get_component(DHT_COMPONENT).peer_manager
|
||||
self.component_manager.peer_manager
|
||||
)
|
||||
|
||||
try:
|
||||
log.info("Peer protocol listening on TCP %i (ext port %i)", peer_port,
|
||||
upnp.upnp_redirects.get("TCP", peer_port))
|
||||
self.lbry_server_port = yield reactor.listenTCP(peer_port, server_factory)
|
||||
self.lbry_server_port = reactor.listenTCP(peer_port, server_factory)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. Visit lbry.io/faq/how-to-change-port for"
|
||||
@@ -617,47 +547,43 @@ class PeerProtocolServerComponent(Component):
|
||||
log.error("%s", traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running on your computer." % str(e))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
if self.lbry_server_port is not None:
|
||||
self.lbry_server_port, old_port = None, self.lbry_server_port
|
||||
log.info('Stop listening on port %s', old_port.port)
|
||||
yield old_port.stopListening()
|
||||
await d2f(old_port.stopListening())
|
||||
|
||||
|
||||
class ReflectorComponent(Component):
|
||||
component_name = REFLECTOR_COMPONENT
|
||||
depends_on = [DHT_COMPONENT, BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
|
||||
depends_on = [BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.reflector_server_port = GCS('reflector_port')
|
||||
self.reflector_server_port = self.conf.reflector_port
|
||||
self.reflector_server = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.reflector_server
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
log.info("Starting reflector server")
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
|
||||
reflector_factory = reflector_server_factory(dht_node.peer_manager, blob_manager, file_manager)
|
||||
reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager)
|
||||
try:
|
||||
self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory)
|
||||
self.reflector_server = await d2f(reactor.listenTCP(self.reflector_server_port, reflector_factory))
|
||||
log.info('Started reflector on port %s', self.reflector_server_port)
|
||||
except error.CannotListenError as e:
|
||||
log.exception("Couldn't bind reflector to port %d", self.reflector_server_port)
|
||||
raise ValueError("{} lbrynet may already be running on your computer.".format(e))
|
||||
raise ValueError(f"{e} lbrynet may already be running on your computer.")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
if self.reflector_server is not None:
|
||||
log.info("Stopping reflector server")
|
||||
self.reflector_server, p = None, self.reflector_server
|
||||
yield p.stopListening
|
||||
await d2f(p.stopListening())
|
||||
|
||||
|
||||
class UPnPComponent(Component):
|
||||
@@ -665,54 +591,131 @@ class UPnPComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self._int_peer_port = GCS('peer_port')
|
||||
self._int_dht_node_port = GCS('dht_node_port')
|
||||
self.use_upnp = GCS('use_upnp')
|
||||
self._int_peer_port = self.conf.peer_port
|
||||
self._int_dht_node_port = self.conf.dht_node_port
|
||||
self.use_upnp = self.conf.use_upnp
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
self.external_ip = None
|
||||
self._maintain_redirects_task = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup_redirects(self):
|
||||
upnp_redirects = yield DeferredDict({
|
||||
"UDP": self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"),
|
||||
"TCP": self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
})
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
async def _repeatedly_maintain_redirects(self, now=True):
|
||||
while True:
|
||||
if now:
|
||||
await self._maintain_redirects()
|
||||
await asyncio.sleep(360)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
if not self.use_upnp:
|
||||
self.external_ip = CS.get_external_ip()
|
||||
return
|
||||
self.upnp = UPnP(self.component_manager.reactor, try_miniupnpc_fallback=True)
|
||||
try:
|
||||
found = yield self.upnp.discover()
|
||||
except Exception as err:
|
||||
log.warning("upnp discovery failed: %s", err)
|
||||
found = False
|
||||
if found and not self.upnp.miniupnpc_runner:
|
||||
log.info("set up redirects using txupnp")
|
||||
elif found and self.upnp.miniupnpc_runner:
|
||||
log.warning("failed to set up redirect with txupnp, miniupnpc fallback was successful")
|
||||
if found:
|
||||
async def _maintain_redirects(self):
|
||||
# setup the gateway if necessary
|
||||
if not self.upnp:
|
||||
try:
|
||||
self.external_ip = yield self.upnp.get_external_ip()
|
||||
yield self._setup_redirects()
|
||||
self.upnp = await UPnP.discover()
|
||||
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
except Exception as err:
|
||||
log.warning("error trying to set up upnp: %s", err)
|
||||
self.external_ip = CS.get_external_ip()
|
||||
else:
|
||||
self.external_ip = CS.get_external_ip()
|
||||
log.warning("upnp discovery failed: %s", err)
|
||||
self.upnp = None
|
||||
|
||||
def stop(self):
|
||||
return defer.DeferredList(
|
||||
[self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()]
|
||||
)
|
||||
# update the external ip
|
||||
external_ip = None
|
||||
if self.upnp:
|
||||
try:
|
||||
external_ip = await self.upnp.get_external_ip()
|
||||
if external_ip != "0.0.0.0" and not self.external_ip:
|
||||
log.info("got external ip from UPnP: %s", external_ip)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
pass
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
external_ip = await get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
self.external_ip = external_ip
|
||||
assert self.external_ip is not None # TODO: handle going/starting offline
|
||||
|
||||
if not self.upnp_redirects and self.upnp: # setup missing redirects
|
||||
try:
|
||||
log.info("add UPnP port mappings")
|
||||
d = {}
|
||||
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
d["TCP"] = self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
d["UDP"] = self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
upnp_redirects = await gather_dict(d)
|
||||
log.info("set up redirects: %s", upnp_redirects)
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
self.upnp = None
|
||||
return self._maintain_redirects()
|
||||
elif self.upnp: # check existing redirects are still active
|
||||
found = set()
|
||||
mappings = await self.upnp.get_redirects()
|
||||
for mapping in mappings:
|
||||
proto = mapping['NewProtocol']
|
||||
if proto in self.upnp_redirects and mapping['NewExternalPort'] == self.upnp_redirects[proto]:
|
||||
if mapping['NewInternalClient'] == self.upnp.lan_address:
|
||||
found.add(proto)
|
||||
if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
udp_port = await self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
self.upnp_redirects['UDP'] = udp_port
|
||||
log.info("refreshed upnp redirect for dht port: %i", udp_port)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
del self.upnp_redirects['UDP']
|
||||
if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
tcp_port = await self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
self.upnp_redirects['TCP'] = tcp_port
|
||||
log.info("refreshed upnp redirect for peer port: %i", tcp_port)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
del self.upnp_redirects['TCP']
|
||||
if ('TCP' in self.upnp_redirects
|
||||
and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components) and (
|
||||
'UDP' in self.upnp_redirects and DHT_COMPONENT not in self.component_manager.skip_components):
|
||||
if self.upnp_redirects:
|
||||
log.debug("upnp redirects are still active")
|
||||
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = await get_external_ip()
|
||||
return
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
if self.upnp:
|
||||
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
|
||||
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
|
||||
log.error("failed to setup upnp, debugging infomation: %s", self.upnp.zipped_debugging_info)
|
||||
else:
|
||||
success = True
|
||||
if self.upnp_redirects:
|
||||
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
|
||||
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
|
||||
|
||||
async def stop(self):
|
||||
if self.upnp_redirects:
|
||||
await asyncio.wait([
|
||||
self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
|
||||
])
|
||||
if self._maintain_redirects_task is not None and not self._maintain_redirects_task.done():
|
||||
self._maintain_redirects_task.cancel()
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'aioupnp_version': aioupnp_version,
|
||||
'redirects': self.upnp_redirects,
|
||||
'gateway': 'No gateway found' if not self.upnp else self.upnp.gateway.manufacturer_string,
|
||||
'dht_redirect_set': 'UDP' in self.upnp_redirects,
|
||||
'peer_redirect_set': 'TCP' in self.upnp_redirects,
|
||||
'external_ip': self.external_ip
|
||||
}
|
||||
|
||||
|
||||
class ExchangeRateManagerComponent(Component):
|
||||
@@ -726,10 +729,8 @@ class ExchangeRateManagerComponent(Component):
|
||||
def component(self):
|
||||
return self.exchange_rate_manager
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
yield self.exchange_rate_manager.start()
|
||||
async def start(self):
|
||||
self.exchange_rate_manager.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.exchange_rate_manager.stop()
|
||||
async def stop(self):
|
||||
self.exchange_rate_manager.stop()
|
||||
@@ -1,17 +1,17 @@
|
||||
import logging
|
||||
import os
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
from lbryschema.fee import Fee
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.schema.fee import Fee
|
||||
|
||||
from lbrynet.core.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
|
||||
from lbrynet.core.Error import DownloadDataTimeout, DownloadCanceledError, DownloadSDTimeout
|
||||
from lbrynet.core.utils import safe_start_looping_call, safe_stop_looping_call
|
||||
from lbrynet.core.StreamDescriptor import download_sd_blob
|
||||
from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from lbrynet import conf
|
||||
from torba.constants import COIN
|
||||
from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
|
||||
from lbrynet.p2p.Error import DownloadDataTimeout, DownloadCanceledError
|
||||
from lbrynet.p2p.StreamDescriptor import download_sd_blob
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from torba.client.constants import COIN
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
INITIALIZING_CODE = 'initializing'
|
||||
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
|
||||
@@ -31,14 +31,18 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GetStream:
|
||||
def __init__(self, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder, rate_limiter,
|
||||
payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None, timeout=None):
|
||||
|
||||
self.timeout = timeout or conf.settings['download_timeout']
|
||||
self.data_rate = data_rate or conf.settings['data_rate']
|
||||
self.max_key_fee = max_key_fee or conf.settings['max_key_fee'][1]
|
||||
self.disable_max_key_fee = disable_max_key_fee or conf.settings['disable_max_key_fee']
|
||||
self.download_directory = conf.settings['download_directory']
|
||||
def __init__(self, conf: Config, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder,
|
||||
rate_limiter, payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None,
|
||||
timeout=None, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.conf = conf
|
||||
self.reactor = reactor
|
||||
self.timeout = timeout or conf.download_timeout
|
||||
self.data_rate = data_rate or conf.data_rate
|
||||
self.max_key_fee = max_key_fee or conf.max_key_fee
|
||||
self.disable_max_key_fee = disable_max_key_fee or conf.disable_max_key_fee
|
||||
self.download_directory = conf.download_dir
|
||||
self.timeout_counter = 0
|
||||
self.code = None
|
||||
self.sd_hash = None
|
||||
@@ -51,44 +55,17 @@ class GetStream:
|
||||
self.sd_identifier = sd_identifier
|
||||
self.storage = storage
|
||||
self.downloader = None
|
||||
self.checker = LoopingCall(self.check_status)
|
||||
|
||||
# fired when the download is complete
|
||||
self.finished_deferred = None
|
||||
# fired after the metadata and the first data blob have been downloaded
|
||||
self.data_downloading_deferred = defer.Deferred(None)
|
||||
self.wrote_data = False
|
||||
|
||||
@property
|
||||
def download_path(self):
|
||||
return os.path.join(self.download_directory, self.downloader.file_name)
|
||||
|
||||
def _check_status(self, status):
|
||||
if status.num_completed > 0 and not self.data_downloading_deferred.called:
|
||||
self.data_downloading_deferred.callback(True)
|
||||
if self.data_downloading_deferred.called:
|
||||
safe_stop_looping_call(self.checker)
|
||||
else:
|
||||
log.debug("Waiting for stream data (%i seconds)", self.timeout_counter)
|
||||
|
||||
def check_status(self):
|
||||
"""
|
||||
Check if we've got the first data blob in the stream yet
|
||||
"""
|
||||
self.timeout_counter += 1
|
||||
if self.timeout_counter > self.timeout:
|
||||
if not self.data_downloading_deferred.called:
|
||||
if self.downloader:
|
||||
err = DownloadDataTimeout(self.sd_hash)
|
||||
else:
|
||||
err = DownloadSDTimeout(self.sd_hash)
|
||||
self.data_downloading_deferred.errback(err)
|
||||
safe_stop_looping_call(self.checker)
|
||||
elif self.downloader:
|
||||
d = self.downloader.status()
|
||||
d.addCallback(self._check_status)
|
||||
else:
|
||||
log.debug("Waiting for stream descriptor (%i seconds)", self.timeout_counter)
|
||||
|
||||
def convert_max_fee(self):
|
||||
currency, amount = self.max_key_fee['currency'], self.max_key_fee['amount']
|
||||
return self.exchange_rate_manager.convert_currency(currency, "LBC", amount)
|
||||
@@ -102,11 +79,11 @@ class GetStream:
|
||||
max_key_fee_amount = self.convert_max_fee()
|
||||
converted_fee_amount = self.exchange_rate_manager.convert_currency(fee.currency, "LBC",
|
||||
fee.amount)
|
||||
if converted_fee_amount > (yield self.wallet.default_account.get_balance()):
|
||||
if converted_fee_amount > (yield f2d(self.wallet.default_account.get_balance())):
|
||||
raise InsufficientFundsError('Unable to pay the key fee of %s' % converted_fee_amount)
|
||||
if converted_fee_amount > max_key_fee_amount and not self.disable_max_key_fee:
|
||||
raise KeyFeeAboveMaxAllowed('Key fee %s above max allowed %s' % (converted_fee_amount,
|
||||
max_key_fee_amount))
|
||||
raise KeyFeeAboveMaxAllowed('Key fee {} above max allowed {}'.format(converted_fee_amount,
|
||||
max_key_fee_amount))
|
||||
converted_fee = {
|
||||
'currency': 'LBC',
|
||||
'amount': converted_fee_amount,
|
||||
@@ -118,7 +95,7 @@ class GetStream:
|
||||
for factory in factories:
|
||||
if isinstance(factory, ManagedEncryptedFileDownloaderFactory):
|
||||
return factory
|
||||
raise Exception('No suitable factory was found in {}'.format(factories))
|
||||
raise Exception(f'No suitable factory was found in {factories}')
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_downloader(self, factory, stream_metadata, file_name=None):
|
||||
@@ -134,11 +111,13 @@ class GetStream:
|
||||
defer.returnValue(downloader)
|
||||
|
||||
def _pay_key_fee(self, address, fee_lbc, name):
|
||||
log.info("Pay key fee %f --> %s", fee_lbc, address)
|
||||
log.info("Pay key fee %s --> %s", dewies_to_lbc(fee_lbc), address)
|
||||
reserved_points = self.wallet.reserve_points(address, fee_lbc)
|
||||
if reserved_points is None:
|
||||
raise InsufficientFundsError('Unable to pay the key fee of %s for %s' % (fee_lbc, name))
|
||||
return self.wallet.send_points_to_address(reserved_points, fee_lbc)
|
||||
raise InsufficientFundsError(
|
||||
'Unable to pay the key fee of {} for {}'.format(dewies_to_lbc(fee_lbc), name)
|
||||
)
|
||||
return f2d(self.wallet.send_points_to_address(reserved_points, fee_lbc))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def pay_key_fee(self, fee, name):
|
||||
@@ -147,18 +126,13 @@ class GetStream:
|
||||
else:
|
||||
defer.returnValue(None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def finish(self, results, name):
|
||||
self.set_status(DOWNLOAD_STOPPED_CODE, name)
|
||||
log.info("Finished downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6],
|
||||
self.download_path)
|
||||
safe_stop_looping_call(self.checker)
|
||||
status = yield self.downloader.status()
|
||||
self._check_status(status)
|
||||
defer.returnValue(self.download_path)
|
||||
return defer.succeed(self.download_path)
|
||||
|
||||
def fail(self, err):
|
||||
safe_stop_looping_call(self.checker)
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -180,8 +154,8 @@ class GetStream:
|
||||
@defer.inlineCallbacks
|
||||
def _download_sd_blob(self):
|
||||
sd_blob = yield download_sd_blob(
|
||||
self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
|
||||
self.wallet, self.timeout, conf.settings['download_mirrors']
|
||||
self.conf, self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
|
||||
self.wallet, self.timeout, self.conf.download_mirrors
|
||||
)
|
||||
defer.returnValue(sd_blob)
|
||||
|
||||
@@ -189,9 +163,11 @@ class GetStream:
|
||||
def _download(self, sd_blob, name, key_fee, txid, nout, file_name=None):
|
||||
self.downloader = yield self._create_downloader(sd_blob, file_name=file_name)
|
||||
yield self.pay_key_fee(key_fee, name)
|
||||
yield self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout))
|
||||
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
|
||||
yield f2d(self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout)))
|
||||
self.finished_deferred = self.downloader.start()
|
||||
self.downloader.download_manager.progress_manager.wrote_first_data.addCallback(
|
||||
self.data_downloading_deferred.callback
|
||||
)
|
||||
self.finished_deferred.addCallbacks(lambda result: self.finish(result, name), self.fail)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -207,16 +183,19 @@ class GetStream:
|
||||
"""
|
||||
self.set_status(INITIALIZING_CODE, name)
|
||||
key_fee = yield self._initialize(stream_info)
|
||||
|
||||
safe_start_looping_call(self.checker, 1)
|
||||
self.set_status(DOWNLOAD_METADATA_CODE, name)
|
||||
try:
|
||||
sd_blob = yield self._download_sd_blob()
|
||||
yield self._download(sd_blob, name, key_fee, txid, nout, file_name)
|
||||
self.set_status(DOWNLOAD_RUNNING_CODE, name)
|
||||
yield self.data_downloading_deferred
|
||||
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
|
||||
self.data_downloading_deferred.addTimeout(self.timeout, self.reactor)
|
||||
try:
|
||||
yield self.data_downloading_deferred
|
||||
self.wrote_data = True
|
||||
except defer.TimeoutError:
|
||||
raise DownloadDataTimeout("data download timed out")
|
||||
except (DownloadDataTimeout, InvalidStreamDescriptorError) as err:
|
||||
safe_stop_looping_call(self.checker)
|
||||
raise err
|
||||
|
||||
defer.returnValue((self.downloader, self.finished_deferred))
|
||||
@@ -2,28 +2,30 @@ import binascii
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer, task
|
||||
from lbrynet.core import utils
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DHTHashAnnouncer:
|
||||
def __init__(self, dht_node, storage, concurrent_announcers=None):
|
||||
def __init__(self, conf: Config, dht_node, storage, concurrent_announcers=None):
|
||||
self.conf = conf
|
||||
self.dht_node = dht_node
|
||||
self.storage = storage
|
||||
self.clock = dht_node.clock
|
||||
self.peer_port = dht_node.peerPort
|
||||
self.hash_queue = []
|
||||
if concurrent_announcers is None:
|
||||
self.concurrent_announcers = conf.settings['concurrent_announcers']
|
||||
self.concurrent_announcers = conf.concurrent_announcers
|
||||
else:
|
||||
self.concurrent_announcers = concurrent_announcers
|
||||
self._manage_lc = None
|
||||
if self.concurrent_announcers:
|
||||
self._manage_lc = task.LoopingCall(self.manage)
|
||||
self._manage_lc.clock = self.clock
|
||||
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.settings['concurrent_announcers'] or 1)
|
||||
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.concurrent_announcers or 1)
|
||||
|
||||
def start(self):
|
||||
if self._manage_lc:
|
||||
@@ -39,7 +41,7 @@ class DHTHashAnnouncer:
|
||||
now = self.clock.seconds()
|
||||
if storing_node_ids:
|
||||
result = (now, storing_node_ids)
|
||||
yield self.storage.update_last_announced_blob(blob_hash, now)
|
||||
yield f2d(self.storage.update_last_announced_blob(blob_hash, now))
|
||||
log.debug("Stored %s to %i peers", blob_hash[:16], len(storing_node_ids))
|
||||
else:
|
||||
result = (None, [])
|
||||
@@ -83,7 +85,7 @@ class DHTHashAnnouncer:
|
||||
if not self.dht_node.contacts:
|
||||
log.info("Not ready to start announcing hashes")
|
||||
return
|
||||
need_reannouncement = yield self.storage.get_blobs_to_announce()
|
||||
need_reannouncement = yield f2d(self.storage.get_blobs_to_announce())
|
||||
if need_reannouncement:
|
||||
yield self.immediate_announce(need_reannouncement)
|
||||
else:
|
||||
@@ -0,0 +1,70 @@
|
||||
import binascii
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DummyPeerFinder:
|
||||
"""This class finds peers which have announced to the DHT that they have certain blobs"""
|
||||
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
|
||||
return defer.succeed([])
|
||||
|
||||
|
||||
class DHTPeerFinder(DummyPeerFinder):
|
||||
"""This class finds peers which have announced to the DHT that they have certain blobs"""
|
||||
#implements(IPeerFinder)
|
||||
|
||||
def __init__(self, component_manager):
|
||||
"""
|
||||
component_manager - an instance of ComponentManager
|
||||
"""
|
||||
self.component_manager = component_manager
|
||||
self.peer_manager = component_manager.peer_manager
|
||||
self.peers = {}
|
||||
self._ongoing_searchs = {}
|
||||
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
|
||||
"""
|
||||
Find peers for blob in the DHT
|
||||
blob_hash (str): blob hash to look for
|
||||
timeout (int): seconds to timeout after
|
||||
filter_self (bool): if True, and if a peer for a blob is itself, filter it
|
||||
from the result
|
||||
|
||||
Returns:
|
||||
list of peers for the blob
|
||||
"""
|
||||
if "dht" in self.component_manager.skip_components:
|
||||
return defer.succeed([])
|
||||
if not self.component_manager.all_components_running("dht"):
|
||||
return defer.succeed([])
|
||||
dht_node = self.component_manager.get_component("dht")
|
||||
|
||||
self.peers.setdefault(blob_hash, {(dht_node.externalIP, dht_node.peerPort,)})
|
||||
if not blob_hash in self._ongoing_searchs or self._ongoing_searchs[blob_hash].called:
|
||||
self._ongoing_searchs[blob_hash] = self._execute_peer_search(dht_node, blob_hash, timeout)
|
||||
|
||||
def _filter_self(blob_hash):
|
||||
my_host, my_port = dht_node.externalIP, dht_node.peerPort
|
||||
return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)}
|
||||
|
||||
peers = set(_filter_self(blob_hash) if filter_self else self.peers[blob_hash])
|
||||
return defer.succeed([self.peer_manager.get_peer(*peer) for peer in peers])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _execute_peer_search(self, dht_node, blob_hash, timeout):
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
finished_deferred = dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
|
||||
timeout = timeout or self.component_manager.conf.peer_search_timeout
|
||||
if timeout:
|
||||
finished_deferred.addTimeout(timeout, dht_node.clock)
|
||||
try:
|
||||
peer_list = yield finished_deferred
|
||||
self.peers[blob_hash].update({(host, port) for _, host, port in peer_list})
|
||||
except defer.TimeoutError:
|
||||
log.debug("DHT timed out while looking peers for blob %s after %s seconds", blob_hash, timeout)
|
||||
finally:
|
||||
del self._ongoing_searchs[blob_hash]
|
||||
@@ -1,4 +1,4 @@
|
||||
from lbrynet.core.Peer import Peer
|
||||
from lbrynet.p2p.Peer import Peer
|
||||
|
||||
|
||||
class PeerManager:
|
||||
@@ -1,16 +1,21 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.file_manager.EncryptedFileCreator import create_lbry_file
|
||||
from lbrynet.blob.EncryptedFileCreator import create_lbry_file
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def d2f(d):
|
||||
return d.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
class Publisher:
|
||||
def __init__(self, blob_manager, payment_rate_manager, storage, lbry_file_manager, wallet, certificate):
|
||||
def __init__(self, account, blob_manager, payment_rate_manager, storage,
|
||||
lbry_file_manager, wallet, certificate):
|
||||
self.account = account
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.storage = storage
|
||||
@@ -19,60 +24,54 @@ class Publisher:
|
||||
self.certificate = certificate
|
||||
self.lbry_file = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def create_and_publish_stream(self, name, bid, claim_dict, file_path, holding_address=None):
|
||||
async def create_and_publish_stream(self, name, bid, claim_dict, file_path, holding_address=None):
|
||||
"""Create lbry file and make claim"""
|
||||
log.info('Starting publish for %s', name)
|
||||
if not os.path.isfile(file_path):
|
||||
raise Exception("File {} not found".format(file_path))
|
||||
raise Exception(f"File {file_path} not found")
|
||||
if os.path.getsize(file_path) == 0:
|
||||
raise Exception("Cannot publish empty file {}".format(file_path))
|
||||
raise Exception(f"Cannot publish empty file {file_path}")
|
||||
|
||||
file_name = os.path.basename(file_path)
|
||||
with open(file_path, 'rb') as read_handle:
|
||||
self.lbry_file = yield create_lbry_file(
|
||||
self.lbry_file = await d2f(create_lbry_file(
|
||||
self.blob_manager, self.storage, self.payment_rate_manager, self.lbry_file_manager, file_name,
|
||||
read_handle
|
||||
)
|
||||
))
|
||||
|
||||
if 'source' not in claim_dict['stream']:
|
||||
claim_dict['stream']['source'] = {}
|
||||
claim_dict['stream']['source']['source'] = self.lbry_file.sd_hash
|
||||
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
|
||||
claim_dict['stream']['source']['contentType'] = get_content_type(file_path)
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
|
||||
tx = yield self.wallet.claim_name(
|
||||
name, bid, claim_dict, self.certificate, holding_address
|
||||
tx = await self.wallet.claim_name(
|
||||
self.account, name, bid, claim_dict, self.certificate, holding_address
|
||||
)
|
||||
|
||||
# check if we have a file already for this claim (if this is a publish update with a new stream)
|
||||
old_stream_hashes = yield self.storage.get_old_stream_hashes_for_claim_id(
|
||||
old_stream_hashes = await self.storage.get_old_stream_hashes_for_claim_id(
|
||||
tx.outputs[0].claim_id, self.lbry_file.stream_hash
|
||||
)
|
||||
if old_stream_hashes:
|
||||
for lbry_file in filter(lambda l: l.stream_hash in old_stream_hashes,
|
||||
list(self.lbry_file_manager.lbry_files)):
|
||||
yield self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False)
|
||||
await d2f(self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False))
|
||||
log.info("Removed old stream for claim update: %s", lbry_file.stream_hash)
|
||||
|
||||
yield self.storage.save_content_claim(
|
||||
await self.storage.save_content_claim(
|
||||
self.lbry_file.stream_hash, tx.outputs[0].id
|
||||
)
|
||||
defer.returnValue(tx)
|
||||
return tx
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def publish_stream(self, name, bid, claim_dict, stream_hash, holding_address=None):
|
||||
async def publish_stream(self, name, bid, claim_dict, stream_hash, holding_address=None):
|
||||
"""Make a claim without creating a lbry file"""
|
||||
tx = yield self.wallet.claim_name(
|
||||
name, bid, claim_dict, self.certificate, holding_address
|
||||
tx = await self.wallet.claim_name(
|
||||
self.account, name, bid, claim_dict, self.certificate, holding_address
|
||||
)
|
||||
if stream_hash: # the stream_hash returned from the db will be None if this isn't a stream we have
|
||||
yield self.storage.save_content_claim(
|
||||
stream_hash.decode(), tx.outputs[0].id
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
)
|
||||
self.lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == stream_hash][0]
|
||||
defer.returnValue(tx)
|
||||
|
||||
|
||||
def get_content_type(filename):
|
||||
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
|
||||
return tx
|
||||
@@ -0,0 +1 @@
|
||||
from lbrynet.extras.daemon import Components # register Component classes
|
||||
@@ -0,0 +1,250 @@
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import aiohttp
|
||||
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
|
||||
from lbrynet.extras import system_info
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
SERVER_STARTUP_ERROR = 'Server Startup Error'
|
||||
DOWNLOAD_STARTED = 'Download Started'
|
||||
DOWNLOAD_ERRORED = 'Download Errored'
|
||||
DOWNLOAD_FINISHED = 'Download Finished'
|
||||
HEARTBEAT = 'Heartbeat'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
NEW_DOWNLOAD_STAT = 'Download'
|
||||
UPNP_SETUP = "UPnP Setup"
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager:
|
||||
|
||||
def __init__(self, conf: Config, installation_id: str, session_id: str):
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._enabled = conf.share_usage_data
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = self._make_context(system_info.get_platform(), 'torba')
|
||||
self.installation_id = installation_id
|
||||
self.session_id = session_id
|
||||
self.task: asyncio.Task = None
|
||||
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
def start(self):
|
||||
if self._enabled and self.task is None:
|
||||
self.task = asyncio.create_task(self.run())
|
||||
log.info("Start")
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
self.task.cancel()
|
||||
|
||||
async def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
request_kwargs = {
|
||||
'method': 'POST',
|
||||
'url': self.url + endpoint,
|
||||
'headers': {'Connection': 'Close'},
|
||||
'auth': aiohttp.BasicAuth(self._write_key, ''),
|
||||
'json': data,
|
||||
'cookies': self.cookies
|
||||
}
|
||||
try:
|
||||
async with aiohttp.request(**request_kwargs) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except Exception as e:
|
||||
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
|
||||
|
||||
async def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if self._enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post('/track', event)
|
||||
|
||||
async def send_new_download_start(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_success(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
|
||||
'message': str(e),
|
||||
})
|
||||
|
||||
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
await self.track({
|
||||
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
|
||||
'event': NEW_DOWNLOAD_STAT,
|
||||
'properties': self._event_properties({
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
|
||||
'action': action,
|
||||
'error': e,
|
||||
}),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow(),
|
||||
})
|
||||
|
||||
async def send_upnp_setup_success_fail(self, success, status):
|
||||
await self.track(
|
||||
self._event(UPNP_SETUP, {
|
||||
'success': success,
|
||||
'status': status,
|
||||
})
|
||||
)
|
||||
|
||||
async def send_server_startup(self):
|
||||
await self.track(self._event(SERVER_STARTUP))
|
||||
|
||||
async def send_server_startup_success(self):
|
||||
await self.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
|
||||
async def send_server_startup_error(self, message):
|
||||
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
async def send_download_started(self, id_, name, claim_dict=None):
|
||||
await self.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
async def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
async def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
async def send_claim_action(self, action):
|
||||
await self.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
async def send_new_channel(self):
|
||||
await self.track(self._event(NEW_CHANNEL))
|
||||
|
||||
async def send_credits_sent(self):
|
||||
await self.track(self._event(CREDITS_SENT))
|
||||
|
||||
async def _send_heartbeat(self):
|
||||
await self.track(self._event(HEARTBEAT))
|
||||
|
||||
async def _update_tracked_metrics(self):
|
||||
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
await self.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self._tracked_data[metric].append(value)
|
||||
|
||||
def summarize_and_reset(self, metric, op=sum):
|
||||
"""Apply `op` on the current values for `metric`.
|
||||
|
||||
This operation also resets the metric.
|
||||
|
||||
Returns:
|
||||
a tuple (should_send, value)
|
||||
"""
|
||||
try:
|
||||
values = self._tracked_data.pop(metric)
|
||||
return True, op(values)
|
||||
except KeyError:
|
||||
return False, None
|
||||
|
||||
def _event(self, event, event_properties=None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': self._event_properties(event_properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _metric_event(self, metric_name, value):
|
||||
return self._event(metric_name, {'value': value})
|
||||
|
||||
def _event_properties(self, event_properties=None):
|
||||
properties = {
|
||||
'lbry_id': self.installation_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
@staticmethod
|
||||
def _download_properties(id_, name, claim_dict=None, report=None):
|
||||
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
|
||||
p = {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
if report:
|
||||
p['report'] = report
|
||||
return p
|
||||
|
||||
@staticmethod
|
||||
def _download_error_properties(error, id_, name, claim_dict, report):
|
||||
def error_name(err):
|
||||
if not hasattr(type(err), "__name__"):
|
||||
return str(type(err))
|
||||
return type(err).__name__
|
||||
return {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': claim_dict.source_hash.decode(),
|
||||
'error': error_name(error),
|
||||
'reason': str(error),
|
||||
'report': report
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _make_context(platform, wallet):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
@@ -1,12 +1,10 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import time
|
||||
import logging
|
||||
import json
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
from lbrynet.core.Error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
from lbrynet.p2p.Error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -38,14 +36,14 @@ class MarketFeed:
|
||||
REQUESTS_TIMEOUT = 20
|
||||
EXCHANGE_RATE_UPDATE_RATE_SEC = 300
|
||||
|
||||
def __init__(self, market, name, url, params, fee):
|
||||
def __init__(self, market: str, name: str, url: str, params, fee):
|
||||
self.market = market
|
||||
self.name = name
|
||||
self.url = url
|
||||
self.params = params
|
||||
self.fee = fee
|
||||
self.rate = None
|
||||
self._updater = LoopingCall(self._update_price)
|
||||
self._task: asyncio.Task = None
|
||||
self._online = True
|
||||
|
||||
def rate_is_initialized(self):
|
||||
@@ -54,17 +52,16 @@ class MarketFeed:
|
||||
def is_online(self):
|
||||
return self._online
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _make_request(self):
|
||||
response = yield treq.get(self.url, params=self.params, timeout=self.REQUESTS_TIMEOUT)
|
||||
defer.returnValue((yield response.content()))
|
||||
async def _make_request(self):
|
||||
async with aiohttp.request('get', self.url, params=self.params) as response:
|
||||
return (await response.read()).decode()
|
||||
|
||||
def _handle_response(self, response):
|
||||
return NotImplementedError
|
||||
raise NotImplementedError()
|
||||
|
||||
def _subtract_fee(self, from_amount):
|
||||
# increase amount to account for market fees
|
||||
return defer.succeed(from_amount / (1.0 - self.fee))
|
||||
return from_amount / (1.0 - self.fee)
|
||||
|
||||
def _save_price(self, price):
|
||||
log.debug("Saving price update %f for %s from %s" % (price, self.market, self.name))
|
||||
@@ -77,21 +74,23 @@ class MarketFeed:
|
||||
log.debug("Exchange rate error (%s from %s): %s", self.market, self.name, err)
|
||||
self._online = False
|
||||
|
||||
def _update_price(self):
|
||||
d = self._make_request()
|
||||
d.addCallback(self._handle_response)
|
||||
d.addCallback(self._subtract_fee)
|
||||
d.addCallback(self._save_price)
|
||||
d.addErrback(self._on_error)
|
||||
return d
|
||||
async def _update_price(self):
|
||||
while True:
|
||||
try:
|
||||
response = await asyncio.wait_for(self._make_request(), self.REQUESTS_TIMEOUT)
|
||||
self._save_price(self._subtract_fee(self._handle_response(response)))
|
||||
except (asyncio.TimeoutError, InvalidExchangeRateResponse) as err:
|
||||
self._on_error(err)
|
||||
await asyncio.sleep(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
|
||||
|
||||
def start(self):
|
||||
if not self._updater.running:
|
||||
self._updater.start(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
|
||||
if not self._task:
|
||||
self._task = asyncio.create_task(self._update_price())
|
||||
|
||||
def stop(self):
|
||||
if self._updater.running:
|
||||
self._updater.stop()
|
||||
if self._task and not self._task.done():
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
|
||||
|
||||
class BittrexFeed(MarketFeed):
|
||||
@@ -116,7 +115,7 @@ class BittrexFeed(MarketFeed):
|
||||
if totals <= 0 or qtys <= 0:
|
||||
raise InvalidExchangeRateResponse(self.market, 'quantities were not positive')
|
||||
vwap = totals / qtys
|
||||
return defer.succeed(float(1.0 / vwap))
|
||||
return float(1.0 / vwap)
|
||||
|
||||
|
||||
class LBRYioFeed(MarketFeed):
|
||||
@@ -133,7 +132,7 @@ class LBRYioFeed(MarketFeed):
|
||||
json_response = json.loads(response)
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(1.0 / json_response['data']['lbc_btc'])
|
||||
return 1.0 / json_response['data']['lbc_btc']
|
||||
|
||||
|
||||
class LBRYioBTCFeed(MarketFeed):
|
||||
@@ -153,7 +152,7 @@ class LBRYioBTCFeed(MarketFeed):
|
||||
raise InvalidExchangeRateResponse(self.name, "invalid rate response : %s" % response)
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(1.0 / json_response['data']['btc_usd'])
|
||||
return 1.0 / json_response['data']['btc_usd']
|
||||
|
||||
|
||||
class CryptonatorBTCFeed(MarketFeed):
|
||||
@@ -174,7 +173,7 @@ class CryptonatorBTCFeed(MarketFeed):
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(float(json_response['ticker']['price']))
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class CryptonatorFeed(MarketFeed):
|
||||
@@ -195,7 +194,7 @@ class CryptonatorFeed(MarketFeed):
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(float(json_response['ticker']['price']))
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class ExchangeRateManager:
|
||||
@@ -234,7 +233,7 @@ class ExchangeRateManager:
|
||||
return self.convert_currency(
|
||||
market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
|
||||
raise CurrencyConversionError(
|
||||
'Unable to convert {} from {} to {}'.format(amount, from_currency, to_currency))
|
||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||
|
||||
def fee_dict(self):
|
||||
return {market: market.rate.as_dict() for market in self.market_feeds}
|
||||
@@ -0,0 +1,104 @@
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
from binascii import hexlify
|
||||
from datetime import datetime
|
||||
from json import JSONEncoder
|
||||
from ecdsa import BadSignatureError
|
||||
from lbrynet.extras.wallet import MainNetLedger
|
||||
from lbrynet.extras.wallet.transaction import Transaction, Output
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class JSONResponseEncoder(JSONEncoder):
|
||||
|
||||
def __init__(self, *args, ledger: MainNetLedger, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ledger = ledger
|
||||
|
||||
def default(self, obj): # pylint: disable=method-hidden
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
if isinstance(obj, Output):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, datetime):
|
||||
return obj.strftime("%Y%m%dT%H:%M:%S")
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
if isinstance(obj, bytes):
|
||||
return obj.decode()
|
||||
return super().default(obj)
|
||||
|
||||
def encode_transaction(self, tx):
|
||||
return {
|
||||
'txid': tx.id,
|
||||
'height': tx.height,
|
||||
'inputs': [self.encode_input(txo) for txo in tx.inputs],
|
||||
'outputs': [self.encode_output(txo) for txo in tx.outputs],
|
||||
'total_input': dewies_to_lbc(tx.input_sum),
|
||||
'total_output': dewies_to_lbc(tx.input_sum - tx.fee),
|
||||
'total_fee': dewies_to_lbc(tx.fee),
|
||||
'hex': hexlify(tx.raw).decode(),
|
||||
}
|
||||
|
||||
def encode_output(self, txo):
|
||||
tx_height = txo.tx_ref.height
|
||||
best_height = self.ledger.headers.height
|
||||
output = {
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'address': txo.get_address(self.ledger),
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
|
||||
}
|
||||
if txo.is_change is not None:
|
||||
output['is_change'] = txo.is_change
|
||||
if txo.is_my_account is not None:
|
||||
output['is_mine'] = txo.is_my_account
|
||||
|
||||
if txo.script.is_claim_involved:
|
||||
output.update({
|
||||
'name': txo.claim_name,
|
||||
'claim_id': txo.claim_id,
|
||||
'permanent_url': txo.permanent_url,
|
||||
})
|
||||
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
claim = txo.claim
|
||||
output['value'] = claim.claim_dict
|
||||
if claim.has_signature:
|
||||
output['valid_signature'] = None
|
||||
if txo.channel is not None:
|
||||
output['channel_name'] = txo.channel.claim_name
|
||||
try:
|
||||
output['valid_signature'] = claim.validate_signature(
|
||||
txo.get_address(self.ledger), txo.channel.claim, name=txo.claim_name
|
||||
)
|
||||
except BadSignatureError:
|
||||
output['valid_signature'] = False
|
||||
except ValueError:
|
||||
log.exception(
|
||||
'txo.id: %s, txo.channel.id:%s, output: %s',
|
||||
txo.id, txo.channel.id, output
|
||||
)
|
||||
output['valid_signature'] = False
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
elif txo.script.is_update_claim:
|
||||
output['type'] = 'update'
|
||||
elif txo.script.is_support_claim:
|
||||
output['type'] = 'support'
|
||||
else:
|
||||
output['type'] = 'basic'
|
||||
|
||||
return output
|
||||
|
||||
def encode_input(self, txi):
|
||||
return self.encode_output(txi.txo_ref.txo) if txi.txo_ref.txo is not None else {
|
||||
'txid': txi.txo_ref.tx_ref.id,
|
||||
'nout': txi.txo_ref.position
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
from lbrynet import utils, __version__
|
||||
|
||||
|
||||
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
class HTTPSLogglyHandler(logging.Handler):
|
||||
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
|
||||
super().__init__()
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.cookies = cookies or {}
|
||||
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
|
||||
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
|
||||
)
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
async def _emit(self, record):
|
||||
payload = self.format(record)
|
||||
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
|
||||
def emit(self, record):
|
||||
asyncio.ensure_future(self._emit(record))
|
||||
|
||||
|
||||
def get_loggly_handler(loggly_token=LOGGLY_TOKEN):
|
||||
handler = HTTPSLogglyHandler(loggly_token)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
return handler
|
||||
@@ -3,29 +3,29 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def migrate_db(db_dir, start, end):
|
||||
def migrate_db(conf, start, end):
|
||||
current = start
|
||||
while current < end:
|
||||
if current == 1:
|
||||
from lbrynet.database.migrator.migrate1to2 import do_migration
|
||||
from .migrate1to2 import do_migration
|
||||
elif current == 2:
|
||||
from lbrynet.database.migrator.migrate2to3 import do_migration
|
||||
from .migrate2to3 import do_migration
|
||||
elif current == 3:
|
||||
from lbrynet.database.migrator.migrate3to4 import do_migration
|
||||
from .migrate3to4 import do_migration
|
||||
elif current == 4:
|
||||
from lbrynet.database.migrator.migrate4to5 import do_migration
|
||||
from .migrate4to5 import do_migration
|
||||
elif current == 5:
|
||||
from lbrynet.database.migrator.migrate5to6 import do_migration
|
||||
from .migrate5to6 import do_migration
|
||||
elif current == 6:
|
||||
from lbrynet.database.migrator.migrate6to7 import do_migration
|
||||
from .migrate6to7 import do_migration
|
||||
elif current == 7:
|
||||
from lbrynet.database.migrator.migrate7to8 import do_migration
|
||||
from .migrate7to8 import do_migration
|
||||
elif current == 8:
|
||||
from lbrynet.database.migrator.migrate8to9 import do_migration
|
||||
from .migrate8to9 import do_migration
|
||||
else:
|
||||
raise Exception("DB migration of version {} to {} is not available".format(current,
|
||||
current+1))
|
||||
do_migration(db_dir)
|
||||
do_migration(conf)
|
||||
current += 1
|
||||
log.info("successfully migrated the database from revision %i to %i", current - 1, current)
|
||||
return None
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
UNSET_NOUT = -1
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blockchainname_db(db_dir)
|
||||
migrate_blockchainname_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blockchainname_db(db_dir)
|
||||
migrate_blockchainname_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blobs_db(db_dir)
|
||||
migrate_blobs_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
add_lbry_file_metadata(db_dir)
|
||||
add_lbry_file_metadata(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
@@ -2,13 +2,11 @@ import sqlite3
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from lbryschema.decode import smart_decode
|
||||
from lbrynet import conf
|
||||
from binascii import hexlify
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
download_directory = conf.settings['download_directory']
|
||||
|
||||
CREATE_TABLES_QUERY = """
|
||||
pragma foreign_keys=on;
|
||||
pragma journal_mode=WAL;
|
||||
@@ -105,13 +103,13 @@ def verify_sd_blob(sd_hash, blob_dir):
|
||||
return decoded, sd_length
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
new_db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
def do_migration(conf):
|
||||
new_db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(new_db_path)
|
||||
|
||||
metadata_db = sqlite3.connect(os.path.join(db_dir, "blockchainname.db"))
|
||||
lbryfile_db = sqlite3.connect(os.path.join(db_dir, 'lbryfile_info.db'))
|
||||
blobs_db = sqlite3.connect(os.path.join(db_dir, 'blobs.db'))
|
||||
metadata_db = sqlite3.connect(os.path.join(conf.data_dir, "blockchainname.db"))
|
||||
lbryfile_db = sqlite3.connect(os.path.join(conf.data_dir, 'lbryfile_info.db'))
|
||||
blobs_db = sqlite3.connect(os.path.join(conf.data_dir, 'blobs.db'))
|
||||
|
||||
name_metadata_cursor = metadata_db.cursor()
|
||||
lbryfile_cursor = lbryfile_db.cursor()
|
||||
@@ -187,10 +185,14 @@ def do_migration(db_dir):
|
||||
(stream_hash, blob_hash, position, iv)
|
||||
)
|
||||
|
||||
download_dir = conf.download_dir
|
||||
if not isinstance(download_dir, bytes):
|
||||
download_dir = download_dir.encode()
|
||||
|
||||
# insert the file
|
||||
transaction.execute(
|
||||
"insert or ignore into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, stream_name, download_directory.encode('hex'),
|
||||
(stream_hash, stream_name, hexlify(download_dir),
|
||||
data_rate, status)
|
||||
)
|
||||
|
||||
@@ -275,7 +277,7 @@ def do_migration(db_dir):
|
||||
|
||||
# recover damaged streams
|
||||
if damaged_stream_sds:
|
||||
blob_dir = os.path.join(db_dir, "blobfiles")
|
||||
blob_dir = os.path.join(conf.data_dir, "blobfiles")
|
||||
damaged_sds_on_disk = [] if not os.path.isdir(blob_dir) else list({p for p in os.listdir(blob_dir)
|
||||
if p in damaged_stream_sds})
|
||||
for damaged_sd in damaged_sds_on_disk:
|
||||
@@ -313,7 +315,7 @@ def do_migration(db_dir):
|
||||
log.warning("detected a failed previous migration to revision 6, repairing it")
|
||||
connection.close()
|
||||
os.remove(new_db_path)
|
||||
return do_migration(db_dir)
|
||||
return do_migration(conf)
|
||||
raise err
|
||||
|
||||
connection.close()
|
||||
@@ -2,8 +2,8 @@ import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
cursor.executescript("alter table blob add last_announced_time integer;")
|
||||