Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e01fbc6f39 | ||
|
|
60ed4e503f | ||
|
|
aa1405887e | ||
|
|
cced217e43 | ||
|
|
8a3b8d2df1 | ||
|
|
be0bd3bdea | ||
|
|
e01c73a7f8 | ||
|
|
134338db4b | ||
|
|
f47a685734 | ||
|
|
6e29f7e99b | ||
|
|
026d93e92a | ||
|
|
a55802c1de | ||
|
|
894397a327 | ||
|
|
3e3f6c0e22 | ||
|
|
a7610e3d34 | ||
|
|
d0e6e74e1b | ||
|
|
126d71978b | ||
|
|
69bcb052de | ||
|
|
071af12d1a | ||
|
|
b7d62ca084 | ||
|
|
fa97f7bd38 | ||
|
|
4a35247016 | ||
|
|
d0526ad7d5 | ||
|
|
19b529cf38 | ||
|
|
143353be44 | ||
|
|
0373196707 | ||
|
|
7a601db23f | ||
|
|
ed61c7feec | ||
|
|
77ece96356 | ||
|
|
80440fd6bd | ||
|
|
ed6537db79 | ||
|
|
79e4767f69 | ||
|
|
0ee279bfd7 | ||
|
|
5249d2a3ca | ||
|
|
4d5823f473 | ||
|
|
c2b9909397 | ||
|
|
afff362544 | ||
|
|
64eb7dbc49 | ||
|
|
2dcd187d69 | ||
|
|
3d9ea92118 | ||
|
|
13a18d422e | ||
|
|
1bbbbd289c | ||
|
|
e33b74f60c | ||
|
|
cd76e0c296 | ||
|
|
3b61222e5a | ||
|
|
2ca438c36e | ||
|
|
8948d5be86 | ||
|
|
a3e64f9cef | ||
|
|
09afec1961 | ||
|
|
f9c465a636 | ||
|
|
89efe3d1b9 | ||
|
|
94cec1b86d | ||
|
|
d62ac0b608 | ||
|
|
5596eac930 | ||
|
|
59f5381b0a | ||
|
|
65ae1b3f08 | ||
|
|
977a7b34f4 | ||
|
|
c01728a6a7 | ||
|
|
5bb94d744e | ||
|
|
494917158c | ||
|
|
ea15674e62 | ||
|
|
34a409ade5 | ||
|
|
273e614715 | ||
|
|
34a725f7ad | ||
|
|
bb6112276f | ||
|
|
db75f0baef | ||
|
|
30689bbf8d | ||
|
|
4bf7742c19 | ||
|
|
69b259c285 | ||
|
|
2fa5233796 | ||
|
|
a5524d490c | ||
|
|
c713fac2d9 | ||
|
|
7a038bbb98 | ||
|
|
eba88c1df7 | ||
|
|
881e5aba68 | ||
|
|
368f0683ec | ||
|
|
94dc8c8520 | ||
|
|
8593cd362a | ||
|
|
2dbf96da81 | ||
|
|
83c7797187 | ||
|
|
6606deafa5 | ||
|
|
19b0d59159 | ||
|
|
9ccb3fa2a3 | ||
|
|
48533df523 | ||
|
|
1ae1137005 | ||
|
|
d5f2832b2f | ||
|
|
d0230b4893 | ||
|
|
a6cd53b97c | ||
|
|
7a28171a72 | ||
|
|
150bcb1116 | ||
|
|
6fe9c5bf00 | ||
|
|
3a86fc6680 | ||
|
|
2805cbb331 | ||
|
|
68d7a7014c | ||
|
|
883fa267a5 | ||
|
|
5f2c3ac965 | ||
|
|
bf8e32ebd2 | ||
|
|
5971be5bef | ||
|
|
677d76a3e4 | ||
|
|
6ed3207cb7 | ||
|
|
be6448b74f | ||
|
|
b707ee4844 | ||
|
|
0e9888be3b | ||
|
|
68b56b7858 | ||
|
|
d32697f611 | ||
|
|
3c54b2bcd7 | ||
|
|
899b182811 | ||
|
|
0dfb5e008d | ||
|
|
abf94357eb | ||
|
|
36aded3830 | ||
|
|
d7ebf50602 | ||
|
|
523c4c86a9 | ||
|
|
74cccdbfc7 | ||
|
|
70471eebfa | ||
|
|
6339224980 | ||
|
|
fe97f7a119 | ||
|
|
f8ed605da2 | ||
|
|
e38ef2af1f | ||
|
|
5d8e627d02 | ||
|
|
c9d4ad5450 | ||
|
|
31586bf588 | ||
|
|
974d3c83e3 | ||
|
|
8dc8ca382b | ||
|
|
1dbb250413 | ||
|
|
9f9ec692c7 | ||
|
|
2c1b54c76c | ||
|
|
7314494fb9 | ||
|
|
e74a5e8ef3 | ||
|
|
b9af999739 | ||
|
|
2de155f90f | ||
|
|
ab3db2e626 | ||
|
|
f9d26b04d8 | ||
|
|
24e0673198 | ||
|
|
833261801f | ||
|
|
6c8cbc1b9e | ||
|
|
6ae4e68d1c | ||
|
|
d22a2dee33 | ||
|
|
0f1ed589a5 | ||
|
|
b032e99dd1 | ||
|
|
fd7811f38d | ||
|
|
071db3a73d | ||
|
|
950c122850 | ||
|
|
e0386ea9dc | ||
|
|
6c3eee0f0e | ||
|
|
4c23add193 | ||
|
|
248d1a18c7 | ||
|
|
17d1b85423 | ||
|
|
b0deb63129 | ||
|
|
ebdb33bd11 | ||
|
|
69ccbda168 | ||
|
|
a38643cca6 | ||
|
|
712b19605e | ||
|
|
8a5f3842fc | ||
|
|
db3e578bcd | ||
|
|
7e22adff7c | ||
|
|
76fbaac658 | ||
|
|
c4fac29ecb | ||
|
|
33392b3351 | ||
|
|
0613cf2d59 | ||
|
|
a930ca0631 | ||
|
|
63a7271008 | ||
|
|
5ca0ebcbb2 | ||
|
|
2c84b7a7dd | ||
|
|
71a0c704bd | ||
|
|
d30e7cd29e | ||
|
|
aeb7c6fe76 | ||
|
|
134c20da93 | ||
|
|
893b4fc3ca | ||
|
|
9b27e81940 | ||
|
|
72c0acf269 | ||
|
|
7be53ad692 | ||
|
|
16e596ec87 | ||
|
|
3fdcb80960 | ||
|
|
26a3221bac | ||
|
|
eb16cb0aa8 | ||
|
|
b5340505e5 | ||
|
|
4bb8129bb7 | ||
|
|
7f838d3a31 | ||
|
|
c186c31bf7 | ||
|
|
86fb99a37c | ||
|
|
6c52450858 | ||
|
|
fb4e94b04a | ||
|
|
248baf58b4 | ||
|
|
0845d65f4e | ||
|
|
dcc5415648 | ||
|
|
3c38a88b22 | ||
|
|
187b7670b1 | ||
|
|
049c17ff08 | ||
|
|
8de97bb441 | ||
|
|
97c0092923 | ||
|
|
e65e1fb5a2 | ||
|
|
20dd1e55f5 | ||
|
|
6aa709e34e | ||
|
|
e73f121c08 | ||
|
|
3b9d6c4f8e | ||
|
|
69b60c19c3 | ||
|
|
04278cb1ab | ||
|
|
e8dec5b440 | ||
|
|
75f82b107e | ||
|
|
0e90c4b611 | ||
|
|
fb66b68bc8 | ||
|
|
c601677386 | ||
|
|
1fbd1cd04e | ||
|
|
4a71b91fbf | ||
|
|
b8b0d8b1ac | ||
|
|
8ef73d3985 | ||
|
|
05e556a5e3 | ||
|
|
d1da42bea8 | ||
|
|
90685e8357 | ||
|
|
a471d5d167 | ||
|
|
16122299e4 | ||
|
|
9f18da4bae | ||
|
|
1abce52867 | ||
|
|
072e2b65d3 | ||
|
|
f6d11e5467 | ||
|
|
458bddcb65 | ||
|
|
5fad43ff69 | ||
|
|
6992d0d947 | ||
|
|
d1b0180437 | ||
|
|
ba59e126cd | ||
|
|
4e3467bb7e | ||
|
|
4d4c705eb9 | ||
|
|
ce7b3184c4 | ||
|
|
b71faddaac | ||
|
|
1328cb4cce | ||
|
|
946488bce9 | ||
|
|
7713881e71 | ||
|
|
26fcc48c5a | ||
|
|
bf12fb68a2 | ||
|
|
3a4c9ff4f9 | ||
|
|
6180a2917d | ||
|
|
014cbb4623 | ||
|
|
ee98251d79 | ||
|
|
e3d08ec773 | ||
|
|
f03e68e63e | ||
|
|
72090b45dd | ||
|
|
93f9622798 | ||
|
|
16ff4b4e2c | ||
|
|
c90cca4b20 | ||
|
|
98d83ea74f | ||
|
|
a234f92d2c | ||
|
|
8ba3575901 | ||
|
|
a857c83c59 | ||
|
|
a6128ac8a2 | ||
|
|
2693301ad0 | ||
|
|
f7c7cb9535 | ||
|
|
b36a5f9b6b | ||
|
|
4826eb9969 | ||
|
|
c34b150794 | ||
|
|
c4c7670a7c | ||
|
|
0918bac86e | ||
|
|
a60a921706 | ||
|
|
b3ee29b66d | ||
|
|
3a1a9b988c | ||
|
|
12f2bebc46 | ||
|
|
31a3812c8a | ||
|
|
70ec08e30b | ||
|
|
23e8f0d849 | ||
|
|
1cdfcd5b4e | ||
|
|
dc2786fbc0 | ||
|
|
7e39b0c4fe | ||
|
|
4af830c405 | ||
|
|
fe844581fd | ||
|
|
443ce6352b | ||
|
|
158c354a27 | ||
|
|
92d1230a90 | ||
|
|
d64916a061 | ||
|
|
20a0568253 | ||
|
|
5a1f42ee54 | ||
|
|
ab95f14e21 | ||
|
|
bae2676b69 | ||
|
|
ea6869a63e | ||
|
|
953327c150 | ||
|
|
e01ad6bc85 | ||
|
|
47a33009ff | ||
|
|
bc844013a1 | ||
|
|
2fd2319cdb | ||
|
|
a50657c472 | ||
|
|
ef5f6e06ba | ||
|
|
977cf9f506 | ||
|
|
ffae10d11f | ||
|
|
221107e76e | ||
|
|
e74c78b607 | ||
|
|
434c7b57b1 | ||
|
|
9dc4608a62 | ||
|
|
fc69366092 | ||
|
|
9f0b1f3e25 | ||
|
|
3f03a845ec | ||
|
|
bc28976e12 | ||
|
|
8f826098f9 | ||
|
|
0a765e6207 | ||
|
|
f83aa61923 | ||
|
|
f9ff3e5d14 | ||
|
|
b3fde9d78d | ||
|
|
0c999083cc | ||
|
|
561215db8a | ||
|
|
8dbb7f2c78 | ||
|
|
6f7f3bb316 | ||
|
|
8d507c4b73 | ||
|
|
2beb0eda94 | ||
|
|
b7e07b72ca | ||
|
|
fde8718c8c | ||
|
|
b2d1bddcfe | ||
|
|
dd1d1be994 | ||
|
|
a93957178a | ||
|
|
5026d0b2f3 | ||
|
|
cc8b47d882 | ||
|
|
db0b5961b3 | ||
|
|
cb6f7ada42 | ||
|
|
59464099b4 | ||
|
|
aed00b53fd | ||
|
|
cdb0f97cb4 | ||
|
|
33bcc8256c | ||
|
|
0a8d44f12f | ||
|
|
1b74594f5b | ||
|
|
b0718d295e | ||
|
|
9bb91b126d | ||
|
|
6dbb00d8d5 | ||
|
|
3a27a4509a | ||
|
|
1d438f5d23 | ||
|
|
3840f19869 | ||
|
|
ec0483b975 | ||
|
|
2fba1f417f | ||
|
|
5b419b55c5 | ||
|
|
b31379e51c | ||
|
|
8394dcb663 | ||
|
|
457a56ce04 | ||
|
|
7c8016d807 | ||
|
|
d704336f20 | ||
|
|
1b4230b06e | ||
|
|
88095af1cd | ||
|
|
e9efd9f68a | ||
|
|
7b32e2ff7c | ||
|
|
572777305e | ||
|
|
7e27a6e37d | ||
|
|
b98621d7ea | ||
|
|
bc5824b8bc | ||
|
|
a6b643bb34 | ||
|
|
488b976734 | ||
|
|
253912b52e | ||
|
|
0bf8416d9f | ||
|
|
2e921437d8 | ||
|
|
b6f9b60683 | ||
|
|
57430e9f5f | ||
|
|
1e13fbee8d | ||
|
|
3d8b9e2b66 | ||
|
|
64898d43f1 | ||
|
|
b2117fddab | ||
|
|
5658623f29 | ||
|
|
934682bd0f | ||
|
|
daecc582f3 | ||
|
|
3cb4012dbf | ||
|
|
f7bd44a4f5 | ||
|
|
52317b05bb | ||
|
|
83c0baa3cb | ||
|
|
340c5addab | ||
|
|
e3fd94dbf0 | ||
|
|
923fcfb9e9 | ||
|
|
0ae5f1a4c6 | ||
|
|
90e06abba2 | ||
|
|
6ecfc94ff1 | ||
|
|
60c4a727d3 | ||
|
|
9fbb9f2e98 | ||
|
|
f1feec66c0 | ||
|
|
73b9b983b1 | ||
|
|
588779e3a4 | ||
|
|
b7ae446059 | ||
|
|
db898307b5 | ||
|
|
7e3512cbf2 | ||
|
|
cdd65c6783 | ||
|
|
320ad717cd | ||
|
|
264e6744e1 | ||
|
|
64e2b79c07 | ||
|
|
9255bb5abe | ||
|
|
d7525dacc3 | ||
|
|
5cb07100f8 | ||
|
|
171c080d33 | ||
|
|
cfc3ec2a05 | ||
|
|
1392b8f83b | ||
|
|
8b265801df | ||
|
|
6c4fb8dab1 | ||
|
|
c1a9cd93ee | ||
|
|
0d4e83e167 | ||
|
|
1292eb345d | ||
|
|
f4c6dc90c9 | ||
|
|
8c8f5d8f58 | ||
|
|
b6d986cb40 | ||
|
|
6cc134c7c8 | ||
|
|
eee964cfa0 | ||
|
|
a03c94bfad | ||
|
|
f1f0917063 | ||
|
|
fe2476b5aa | ||
|
|
9c5fa8bf76 | ||
|
|
cd43a71172 | ||
|
|
e2c6de76f4 | ||
|
|
05aeadc9fe | ||
|
|
7153417372 | ||
|
|
fceb2ae0a2 | ||
|
|
0cef2fdd82 | ||
|
|
9c91b135bd | ||
|
|
e3b127192c | ||
|
|
ee1738da6e | ||
|
|
8ca2b75c80 | ||
|
|
dc81b6def1 | ||
|
|
22f5aaee70 | ||
|
|
4bd6569c9c | ||
|
|
dbf39baeb1 | ||
|
|
3adf3c26f2 | ||
|
|
fa0d13e630 | ||
|
|
227a82d6d3 | ||
|
|
65cfc1f63c | ||
|
|
e46f0e2444 | ||
|
|
00bc7995c4 | ||
|
|
2dcebe48b7 | ||
|
|
d12b6cddb3 | ||
|
|
699c057f2d | ||
|
|
3ce9f31f4b | ||
|
|
485d5d55e8 | ||
|
|
6ee9f2513b | ||
|
|
81a6e14a5c | ||
|
|
496ffe0096 | ||
|
|
216c760179 | ||
|
|
01eca3004f | ||
|
|
2f47a3758e | ||
|
|
73bcc1796f | ||
|
|
e03e41ad31 | ||
|
|
6cb30f8e05 | ||
|
|
d9b1be6214 | ||
|
|
5f2f06c269 | ||
|
|
e1dcdcf4db | ||
|
|
416472c5d8 | ||
|
|
196f3ba92d | ||
|
|
6feae93c58 | ||
|
|
203d30d4f7 | ||
|
|
d0a7ca841b | ||
|
|
2e6c848c39 | ||
|
|
18a0204105 | ||
|
|
01baa193d5 | ||
|
|
ba57af5502 | ||
|
|
8139f93ff1 | ||
|
|
4596d5c26f | ||
|
|
810a63782d | ||
|
|
ea83827d57 |
No files matched your search
@@ -0,0 +1,7 @@
|
||||
.git
|
||||
.tox
|
||||
__pycache__
|
||||
dist
|
||||
lbrynet.egg-info
|
||||
docs
|
||||
tests
|
||||
@@ -51,3 +51,24 @@ Tell us what happens instead
|
||||
|
||||
## Screenshots
|
||||
<!-- If a screenshot would help explain the bug, please include one or two here -->
|
||||
|
||||
|
||||
|
||||
## Internal Use
|
||||
|
||||
### Acceptance Criteria
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
### Definition of Done
|
||||
- [ ] Tested against acceptance criteria
|
||||
- [ ] Tested against the assumptions of user story
|
||||
- [ ] The project builds without errors
|
||||
- [ ] Unit tests are written and passing
|
||||
- [ ] Tests on devices/browsers listed in the issue have passed
|
||||
- [ ] QA performed & issues resolved
|
||||
- [ ] Refactoring completed
|
||||
- [ ] Any configuration or build changes documented
|
||||
- [ ] Documentation updated
|
||||
- [ ] Peer Code Review performed
|
||||
@@ -0,0 +1,38 @@
|
||||
## PR Checklist
|
||||
Please check all that apply to this PR using "x":
|
||||
|
||||
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
|
||||
- [ ] I have checked that this PR does not introduce a breaking change
|
||||
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
|
||||
|
||||
|
||||
## PR Type
|
||||
What kind of change does this PR introduce?
|
||||
|
||||
Why is this change necessary?
|
||||
|
||||
<!-- Please check all that apply to this PR using "x". -->
|
||||
|
||||
- [ ] Bugfix
|
||||
- [ ] Feature
|
||||
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
|
||||
- [ ] Code style update (formatting)
|
||||
- [ ] Refactoring (no functional changes)
|
||||
- [ ] Documentation changes
|
||||
- [ ] Other - Please describe:
|
||||
|
||||
## Fixes
|
||||
|
||||
Issue Number: N/A
|
||||
|
||||
|
||||
## What is the current behavior?
|
||||
|
||||
|
||||
## What is the new behavior?
|
||||
|
||||
|
||||
## Other information
|
||||
|
||||
|
||||
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS
|
||||
ignore=CVS,schema
|
||||
|
||||
# Add files or directories matching the regex patterns to the
|
||||
# blacklist. The regex matches against base names, not paths.
|
||||
@@ -354,13 +354,13 @@ ignore-mixin-members=yes
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=twisted.internet.reactor,leveldb,distutils
|
||||
ignored-modules=leveldb,distutils
|
||||
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
|
||||
|
||||
# List of classes names for which member attributes should not be checked
|
||||
# (useful for classes with attributes dynamically set). This supports can work
|
||||
# with qualified names.
|
||||
ignored-classes=twisted.internet.reactor,RequestMessage
|
||||
# ignored-classes=
|
||||
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
|
||||
@@ -9,50 +9,28 @@ jobs:
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid==2.0.4 aiohttp==3.4.4
|
||||
# newer astroid and aiohttp fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git
|
||||
- pip install git+https://github.com/lbryio/lbryschema.git
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
|
||||
- &tests
|
||||
stage: test
|
||||
name: "Unit Tests w/ Python 3.7"
|
||||
- stage: test
|
||||
name: "Unit Tests"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/electrumx.git#lbryumx
|
||||
- pip install git+https://github.com/lbryio/orchstr8.git
|
||||
- pip install git+https://github.com/lbryio/lbryschema.git
|
||||
- pip install git+https://github.com/lbryio/lbryumx.git
|
||||
- pip install git+https://github.com/lbryio/torba.git
|
||||
- pip install -e .[test]
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -v tests.unit.wallet
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics tests.unit.components tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.test_cli tests.unit.test_customLogger
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- <<: *tests
|
||||
name: "Unit Tests w/ Python 3.6"
|
||||
python: "3.6"
|
||||
|
||||
- <<: *tests
|
||||
name: "DHT Tests w/ Python 3.7"
|
||||
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
|
||||
|
||||
- <<: *tests
|
||||
name: "DHT Tests w/ Python 3.6"
|
||||
python: "3.6"
|
||||
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
|
||||
|
||||
- name: "Integration Tests"
|
||||
install:
|
||||
- pip install tox-travis coverage
|
||||
- pushd .. && git clone https://github.com/lbryio/electrumx.git --branch lbryumx && popd
|
||||
- pushd .. && git clone https://github.com/lbryio/orchstr8.git && popd
|
||||
- pushd .. && git clone https://github.com/lbryio/lbryschema.git && popd
|
||||
- pushd .. && git clone https://github.com/lbryio/lbryumx.git && cd lbryumx && git checkout afd34f323dd94c516108a65240f7d17aea8efe85 && cd .. && popd
|
||||
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
|
||||
script: tox
|
||||
after_success:
|
||||
@@ -65,32 +43,50 @@ jobs:
|
||||
services:
|
||||
- docker
|
||||
install:
|
||||
- docker pull cdrx/pyinstaller-windows:python3-32bit
|
||||
- docker pull lbry/pyinstaller34_32bits:py371
|
||||
script:
|
||||
- docker run -v "$(pwd):/src/lbry" cdrx/pyinstaller-windows:python3-32bit lbry/scripts/wine_build.sh
|
||||
- python scripts/set_build.py
|
||||
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
|
||||
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-windows.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
paths:
|
||||
- lbrynet-windows.zip
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})/win/
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- &build
|
||||
name: "Linux"
|
||||
python: "3.6"
|
||||
env: OS=linux
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git
|
||||
- pip3 install git+https://github.com/lbryio/lbryschema.git
|
||||
- python scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbrynet/cli.py
|
||||
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
|
||||
- ./dist/lbrynet --version
|
||||
env: OS=linux
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-${OS}.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
@@ -98,14 +94,44 @@ jobs:
|
||||
- lbrynet-${OS}.zip
|
||||
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})/${OS}/lbrynet
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- <<: *build
|
||||
name: "Mac"
|
||||
os: osx
|
||||
osx_image: xcode9.4
|
||||
osx_image: xcode8.3
|
||||
language: generic
|
||||
env: OS=mac
|
||||
cache: false
|
||||
before_install:
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
- pip3 install pyinstaller
|
||||
- git clone https://github.com/lbryio/torba.git --depth 1
|
||||
- sed -i -e "s/'plyvel',//" torba/setup.py
|
||||
- cd torba && pip3 install -e . && cd ..
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
|
||||
- if: tag IS present
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Tagged Release"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:$TRAVIS_TAG
|
||||
- if: tag IS blank AND branch = master
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Master"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:master
|
||||
|
||||
cache:
|
||||
directories:
|
||||
|
||||
@@ -7,7 +7,47 @@ regard to the json-rpc api. As we're currently pre-1.0 release, we
|
||||
can and probably will change functionality and break backwards compatibility
|
||||
at anytime.
|
||||
|
||||
## [0.30.0] - 2018-10-17
|
||||
|
||||
## [0.30.1] - 2018-12-09
|
||||
|
||||
### API
|
||||
* add `aioupnp_version` to `upnp` field in the response to `status`
|
||||
* fixed uncaught claim signature errors
|
||||
* `--blocking` support for `abandon` command (https://github.com/lbryio/lbry/pull/1618)
|
||||
* don't create SSL keys if SSL is disabled (https://github.com/lbryio/lbry/pull/1644)
|
||||
* added `account_send` command (https://github.com/lbryio/lbry/pull/1654)
|
||||
* fixes for claim send to address (https://github.com/lbryio/lbry/pull/1651)
|
||||
* better handle errors on resolve (https://github.com/lbryio/lbry/pull/1662)
|
||||
* fix `transaction_show` (https://github.com/lbryio/lbry/pull/1689)
|
||||
|
||||
### Schema
|
||||
* merged `lbryschema` repository into `lbrynet` (https://github.com/lbryio/lbry/pull/1444)
|
||||
* updated `protobuf` to `3.2.0` (https://github.com/lbryio/lbry/issues/1602)
|
||||
* use `types` repo for generating python protobufs (https://github.com/lbryio/lbry/pull/1605)
|
||||
|
||||
### Wallet
|
||||
* fixed resolving names with invalid channel signatures
|
||||
* fixed wallet sync issues
|
||||
* fixed wallet locking/encryption issues
|
||||
|
||||
### DHT
|
||||
* fixed retrying kademlia rpc calls (https://github.com/lbryio/lbry/pull/1495)
|
||||
* fixed `KeyError` in `iterativeFindValue`, this is used by `peer_list` and when finding peers to download blobs from (https://github.com/lbryio/lbry/issues/1638)
|
||||
* settings external ip address and ability to disable more DHT components (https://github.com/lbryio/lbry/pull/1604)
|
||||
* improved routing table bucket initialization (https://github.com/lbryio/lbry/pull/1633)
|
||||
|
||||
### Database
|
||||
* resolved too many variables in sqlite issue (https://github.com/lbryio/lbry/pull/1612)
|
||||
|
||||
### Misc
|
||||
* switch to Python 3.7
|
||||
* brought back icon for windows binary
|
||||
* using custom list of mime types with more types (https://github.com/lbryio/lbry/pull/1700)
|
||||
* add balance to startup report (https://github.com/lbryio/lbry/pull/1613)
|
||||
* drop `six` dependency (https://github.com/lbryio/lbry/pull/1660)
|
||||
* drop `mock` dependency (https://github.com/lbryio/lbry/pull/1661)
|
||||
|
||||
## [0.30.0] - 2018-10-22
|
||||
This release is the most ambitious upgrade to lbrynet yet; including a brand
|
||||
new wallet implementation, switch to Python 3 of the entire code base and
|
||||
many changes to the API such as pagination and a more consistent output from
|
||||
@@ -58,6 +98,7 @@ most commands.
|
||||
* removed `send_amount_to_address` command previously marked as deprecated
|
||||
* removed `channel_list_mine` command previously marked as deprecated
|
||||
* removed `get_availability` command previously marked as deprecated
|
||||
* updated `txupnp` dependency to `aioupnp` (UPnP for asyncio)
|
||||
|
||||
### Wallet
|
||||
* changed to a new wallet implementation: [torba](https://github.com/lbryio/torba).
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
# Installing LBRY
|
||||
|
||||
If only the json-rpc API server is needed the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
|
||||
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
|
||||
|
||||
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
|
||||
Here's a video walkthrough of this setup which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch):
|
||||
Here's a video walkthrough of this setup, which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch):
|
||||
[](https://spee.ch/967f99344308f1e90f0620d91b6c93e4dfb240e0/lbrynet-dev-setup.mp4)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred). Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
|
||||
|
||||
After installing python 3 you'll need to install some additional libraries depending on your operating system.
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
### Virtualenv
|
||||
|
||||
@@ -22,9 +22,9 @@ Once python 3 is installed run `python3 -m pip install virtualenv` to install vi
|
||||
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
|
||||
|
||||
|
||||
### OS X
|
||||
### macOS
|
||||
|
||||
OS X users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
@@ -42,7 +42,7 @@ sudo apt-get install build-essential python3.7 python3.7-dev git python-virtuale
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
|
||||
If you're running another Linux flavor, install the equivalent of the above packages for your system.
|
||||
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -51,16 +51,16 @@ To install:
|
||||
```
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
|
||||
|
||||
virtualenv lbry-venv --python=python3.7
|
||||
source lbry-venv/bin/activate
|
||||
|
||||
python --version # Python 2 is not supported. Make sure you're on Python 3.7
|
||||
|
||||
pip install --editable .[test] # [test] installs extras needed for running the tests
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
To verify your install, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
|
||||
## Run the tests
|
||||
To run the unit tests from the repo directory:
|
||||
|
||||
@@ -1,12 +1,19 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized network. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, and rights and access rules. LBRY also provides a data network that consists of peers uploading and downloading data from other peers, possibly in exchange for payments, and a distributed hash table, used by peers to discover other peers.
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
This project provides an SDK for building applications using the LBRY protocol. The SDK is written in Python 3.7+ using Twisted. For other languages or platforms, a JSONRPC API is provided.
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
|
||||
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases, for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
|
||||
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -14,13 +21,15 @@ Run `lbrynet start` to launch the API server.
|
||||
|
||||
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
|
||||
Our [quickstart guide](http://lbry.io/quickstart) provides a simple walkthrough and examples for learning.
|
||||
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
|
||||
|
||||
With the daemon running, `lbrynet commands` will show you a list of commands.
|
||||
|
||||
The full API is documented [here](https://lbry.tech/api/sdk).
|
||||
|
||||
## Running from source
|
||||
|
||||
Installing from source is also relatively painless, full instructions are in [INSTALL.md](INSTALL.md)
|
||||
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
|
||||
|
||||
## Contributing
|
||||
|
||||
@@ -36,8 +45,12 @@ We take security seriously. Please contact security@lbry.io regarding any securi
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@jackrobison](mailto:jack@lbry.io).
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
|
||||
|
||||
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
|
||||
|
||||
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry/blob/master/example_daemon_settings.yml).
|
||||
@@ -0,0 +1,5 @@
|
||||
build:
|
||||
python setup.py apk
|
||||
adb install -r lbry-0.1-service-debug.apk
|
||||
|
||||
.PHONY: build
|
||||
@@ -0,0 +1,49 @@
|
||||
import inspect
|
||||
from os.path import dirname, join
|
||||
|
||||
import sh
|
||||
from pythonforandroid.util import ensure_dir
|
||||
from pythonforandroid.toolchain import (
|
||||
Bootstrap, current_directory, info, info_main, shprint
|
||||
)
|
||||
|
||||
|
||||
class LBRYServiceBootstrap(Bootstrap):
|
||||
name = 'lbry-service'
|
||||
recipe_depends = ['genericndkbuild', 'python3']
|
||||
bootstrap_dir = dirname(__file__)
|
||||
|
||||
def get_common_dir(self):
|
||||
return join(dirname(inspect.getfile(Bootstrap)), 'bootstraps', 'common')
|
||||
|
||||
def run_distribute(self):
|
||||
info_main('# Creating Android project from build and {} bootstrap'.format(
|
||||
self.name))
|
||||
|
||||
info('This currently just copies the build stuff straight from the build dir.')
|
||||
shprint(sh.rm, '-rf', self.dist_dir)
|
||||
shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
|
||||
with current_directory(self.dist_dir):
|
||||
with open('local.properties', 'w') as fileh:
|
||||
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
|
||||
|
||||
arch = self.ctx.archs[0]
|
||||
if len(self.ctx.archs) > 1:
|
||||
raise ValueError('built for more than one arch, but bootstrap cannot handle that yet')
|
||||
info('Bootstrap running with arch {}'.format(arch))
|
||||
|
||||
with current_directory(self.dist_dir):
|
||||
info('Copying python distribution')
|
||||
|
||||
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
|
||||
self.distribute_aars(arch)
|
||||
self.distribute_javaclasses(self.ctx.javaclass_dir)
|
||||
|
||||
python_bundle_dir = join('_python_bundle', '_python_bundle')
|
||||
ensure_dir(python_bundle_dir)
|
||||
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
|
||||
join(self.dist_dir, python_bundle_dir), arch)
|
||||
|
||||
self.strip_libraries(arch)
|
||||
self.fry_eggs(site_packages_dir)
|
||||
super().run_distribute()
|
||||
@@ -0,0 +1,89 @@
|
||||
# prevent user to include invalid extensions
|
||||
*.apk
|
||||
*.pxd
|
||||
|
||||
# eggs
|
||||
*.egg-info
|
||||
|
||||
# unit test
|
||||
unittest/*
|
||||
|
||||
# python config
|
||||
config/makesetup
|
||||
|
||||
# unused kivy files (platform specific)
|
||||
kivy/input/providers/wm_*
|
||||
kivy/input/providers/mactouch*
|
||||
kivy/input/providers/probesysfs*
|
||||
kivy/input/providers/mtdev*
|
||||
kivy/input/providers/hidinput*
|
||||
kivy/core/camera/camera_videocapture*
|
||||
kivy/core/spelling/*osx*
|
||||
kivy/core/video/video_pyglet*
|
||||
kivy/tools
|
||||
kivy/tests/*
|
||||
kivy/*/*.h
|
||||
kivy/*/*.pxi
|
||||
|
||||
# unused encodings
|
||||
lib-dynload/*codec*
|
||||
encodings/cp*.pyo
|
||||
encodings/tis*
|
||||
encodings/shift*
|
||||
encodings/bz2*
|
||||
encodings/iso*
|
||||
encodings/undefined*
|
||||
encodings/johab*
|
||||
encodings/p*
|
||||
encodings/m*
|
||||
encodings/euc*
|
||||
encodings/k*
|
||||
encodings/unicode_internal*
|
||||
encodings/quo*
|
||||
encodings/gb*
|
||||
encodings/big5*
|
||||
encodings/hp*
|
||||
encodings/hz*
|
||||
|
||||
# unused python modules
|
||||
bsddb/*
|
||||
wsgiref/*
|
||||
hotshot/*
|
||||
pydoc_data/*
|
||||
tty.pyo
|
||||
anydbm.pyo
|
||||
nturl2path.pyo
|
||||
LICENCE.txt
|
||||
macurl2path.pyo
|
||||
dummy_threading.pyo
|
||||
audiodev.pyo
|
||||
antigravity.pyo
|
||||
dumbdbm.pyo
|
||||
sndhdr.pyo
|
||||
__phello__.foo.pyo
|
||||
sunaudio.pyo
|
||||
os2emxpath.pyo
|
||||
multiprocessing/dummy*
|
||||
|
||||
# unused binaries python modules
|
||||
lib-dynload/termios.so
|
||||
lib-dynload/_lsprof.so
|
||||
lib-dynload/*audioop.so
|
||||
lib-dynload/_hotshot.so
|
||||
lib-dynload/_heapq.so
|
||||
lib-dynload/_json.so
|
||||
lib-dynload/grp.so
|
||||
lib-dynload/resource.so
|
||||
lib-dynload/pyexpat.so
|
||||
lib-dynload/_ctypes_test.so
|
||||
lib-dynload/_testcapi.so
|
||||
|
||||
# odd files
|
||||
plat-linux3/regen
|
||||
|
||||
#>sqlite3
|
||||
# conditionnal include depending if some recipes are included or not.
|
||||
sqlite3/*
|
||||
lib-dynload/_sqlite3.so
|
||||
#<sqlite3
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
include $(call all-subdir-makefiles)
|
||||
@@ -0,0 +1,7 @@
|
||||
|
||||
# Uncomment this if you're using STL in your project
|
||||
# See CPLUSPLUS-SUPPORT.html in the NDK documentation for more information
|
||||
# APP_STL := stlport_static
|
||||
|
||||
# APP_ABI := armeabi armeabi-v7a x86
|
||||
APP_ABI := $(ARCH)
|
||||
@@ -0,0 +1,22 @@
|
||||
LOCAL_PATH := $(call my-dir)
|
||||
|
||||
include $(CLEAR_VARS)
|
||||
|
||||
LOCAL_MODULE := main
|
||||
|
||||
# Add your application source files here...
|
||||
LOCAL_SRC_FILES := start.c pyjniusjni.c
|
||||
|
||||
LOCAL_CFLAGS += -I$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_INCLUDE_ROOT) $(EXTRA_CFLAGS)
|
||||
|
||||
LOCAL_SHARED_LIBRARIES := python_shared
|
||||
|
||||
LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS)
|
||||
|
||||
LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS)
|
||||
|
||||
include $(BUILD_SHARED_LIBRARY)
|
||||
|
||||
ifdef CRYSTAX_PYTHON_VERSION
|
||||
$(call import-module,python/$(CRYSTAX_PYTHON_VERSION))
|
||||
endif
|
||||
@@ -0,0 +1,10 @@
|
||||
LOCAL_PATH := $(call my-dir)
|
||||
|
||||
include $(CLEAR_VARS)
|
||||
|
||||
LOCAL_MODULE := main
|
||||
|
||||
LOCAL_SRC_FILES := YourSourceHere.c
|
||||
|
||||
include $(BUILD_SHARED_LIBRARY)
|
||||
$(call import-module,SDL)LOCAL_PATH := $(call my-dir)
|
||||
@@ -0,0 +1,6 @@
|
||||
|
||||
#define BOOTSTRAP_NAME_SERVICEONLY
|
||||
#define BOOTSTRAP_USES_NO_SDL_HEADERS
|
||||
|
||||
const char bootstrap_name[] = "service_only";
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
|
||||
#include <pthread.h>
|
||||
#include <jni.h>
|
||||
|
||||
#define LOGI(...) do {} while (0)
|
||||
#define LOGE(...) do {} while (0)
|
||||
|
||||
#include "android/log.h"
|
||||
|
||||
/* These JNI management functions are taken from SDL2, but modified to refer to pyjnius */
|
||||
|
||||
/* #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) */
|
||||
/* #define LOGP(x) LOG("python", (x)) */
|
||||
#define LOG_TAG "Python_android"
|
||||
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
|
||||
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
|
||||
|
||||
|
||||
/* Function headers */
|
||||
JNIEnv* Android_JNI_GetEnv(void);
|
||||
static void Android_JNI_ThreadDestroyed(void*);
|
||||
|
||||
static pthread_key_t mThreadKey;
|
||||
static JavaVM* mJavaVM;
|
||||
|
||||
int Android_JNI_SetupThread(void)
|
||||
{
|
||||
Android_JNI_GetEnv();
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Library init */
|
||||
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
|
||||
{
|
||||
JNIEnv *env;
|
||||
mJavaVM = vm;
|
||||
LOGI("JNI_OnLoad called");
|
||||
if ((*mJavaVM)->GetEnv(mJavaVM, (void**) &env, JNI_VERSION_1_4) != JNI_OK) {
|
||||
LOGE("Failed to get the environment using GetEnv()");
|
||||
return -1;
|
||||
}
|
||||
/*
|
||||
* Create mThreadKey so we can keep track of the JNIEnv assigned to each thread
|
||||
* Refer to http://developer.android.com/guide/practices/design/jni.html for the rationale behind this
|
||||
*/
|
||||
if (pthread_key_create(&mThreadKey, Android_JNI_ThreadDestroyed) != 0) {
|
||||
|
||||
__android_log_print(ANDROID_LOG_ERROR, "pyjniusjni", "Error initializing pthread key");
|
||||
}
|
||||
Android_JNI_SetupThread();
|
||||
|
||||
return JNI_VERSION_1_4;
|
||||
}
|
||||
|
||||
JNIEnv* Android_JNI_GetEnv(void)
|
||||
{
|
||||
/* From http://developer.android.com/guide/practices/jni.html
|
||||
* All threads are Linux threads, scheduled by the kernel.
|
||||
* They're usually started from managed code (using Thread.start), but they can also be created elsewhere and then
|
||||
* attached to the JavaVM. For example, a thread started with pthread_create can be attached with the
|
||||
* JNI AttachCurrentThread or AttachCurrentThreadAsDaemon functions. Until a thread is attached, it has no JNIEnv,
|
||||
* and cannot make JNI calls.
|
||||
* Attaching a natively-created thread causes a java.lang.Thread object to be constructed and added to the "main"
|
||||
* ThreadGroup, making it visible to the debugger. Calling AttachCurrentThread on an already-attached thread
|
||||
* is a no-op.
|
||||
* Note: You can call this function any number of times for the same thread, there's no harm in it
|
||||
*/
|
||||
|
||||
JNIEnv *env;
|
||||
int status = (*mJavaVM)->AttachCurrentThread(mJavaVM, &env, NULL);
|
||||
if(status < 0) {
|
||||
LOGE("failed to attach current thread");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* From http://developer.android.com/guide/practices/jni.html
|
||||
* Threads attached through JNI must call DetachCurrentThread before they exit. If coding this directly is awkward,
|
||||
* in Android 2.0 (Eclair) and higher you can use pthread_key_create to define a destructor function that will be
|
||||
* called before the thread exits, and call DetachCurrentThread from there. (Use that key with pthread_setspecific
|
||||
* to store the JNIEnv in thread-local-storage; that way it'll be passed into your destructor as the argument.)
|
||||
* Note: The destructor is not called unless the stored value is != NULL
|
||||
* Note: You can call this function any number of times for the same thread, there's no harm in it
|
||||
* (except for some lost CPU cycles)
|
||||
*/
|
||||
pthread_setspecific(mThreadKey, (void*) env);
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
static void Android_JNI_ThreadDestroyed(void* value)
|
||||
{
|
||||
/* The thread is being destroyed, detach it from the Java VM and set the mThreadKey value to NULL as required */
|
||||
JNIEnv *env = (JNIEnv*) value;
|
||||
if (env != NULL) {
|
||||
(*mJavaVM)->DetachCurrentThread(mJavaVM);
|
||||
pthread_setspecific(mThreadKey, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
void *WebView_AndroidGetJNIEnv()
|
||||
{
|
||||
return Android_JNI_GetEnv();
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class Octal {
|
||||
|
||||
/**
|
||||
* Parse an octal string from a header buffer. This is used for the file
|
||||
* permission mode value.
|
||||
*
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The long value of the octal string.
|
||||
*/
|
||||
public static long parseOctal(byte[] header, int offset, int length) {
|
||||
long result = 0;
|
||||
boolean stillPadding = true;
|
||||
|
||||
int end = offset + length;
|
||||
for (int i = offset; i < end; ++i) {
|
||||
if (header[i] == 0)
|
||||
break;
|
||||
|
||||
if (header[i] == (byte) ' ' || header[i] == '0') {
|
||||
if (stillPadding)
|
||||
continue;
|
||||
|
||||
if (header[i] == (byte) ' ')
|
||||
break;
|
||||
}
|
||||
|
||||
stillPadding = false;
|
||||
|
||||
result = ( result << 3 ) + ( header[i] - '0' );
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an octal integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The integer value of the octal bytes.
|
||||
*/
|
||||
public static int getOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
int idx = length - 1;
|
||||
|
||||
buf[offset + idx] = 0;
|
||||
--idx;
|
||||
buf[offset + idx] = (byte) ' ';
|
||||
--idx;
|
||||
|
||||
if (value == 0) {
|
||||
buf[offset + idx] = (byte) '0';
|
||||
--idx;
|
||||
} else {
|
||||
for (long val = value; idx >= 0 && val > 0; --idx) {
|
||||
buf[offset + idx] = (byte) ( (byte) '0' + (byte) ( val & 7 ) );
|
||||
val = val >> 3;
|
||||
}
|
||||
}
|
||||
|
||||
for (; idx >= 0; --idx) {
|
||||
buf[offset + idx] = (byte) ' ';
|
||||
}
|
||||
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the checksum octal integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The integer value of the entry's checksum.
|
||||
*/
|
||||
public static int getCheckSumOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
getOctalBytes( value, buf, offset, length );
|
||||
buf[offset + length - 1] = (byte) ' ';
|
||||
buf[offset + length - 2] = 0;
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an octal long integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The long value of the octal bytes.
|
||||
*/
|
||||
public static int getLongOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
byte[] temp = new byte[length + 1];
|
||||
getOctalBytes( value, temp, 0, length + 1 );
|
||||
System.arraycopy( temp, 0, buf, offset, length );
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarConstants {
|
||||
public static final int EOF_BLOCK = 1024;
|
||||
public static final int DATA_BLOCK = 512;
|
||||
public static final int HEADER_BLOCK = 512;
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarEntry {
|
||||
protected File file;
|
||||
protected TarHeader header;
|
||||
|
||||
private TarEntry() {
|
||||
this.file = null;
|
||||
header = new TarHeader();
|
||||
}
|
||||
|
||||
public TarEntry(File file, String entryName) {
|
||||
this();
|
||||
this.file = file;
|
||||
this.extractTarHeader(entryName);
|
||||
}
|
||||
|
||||
public TarEntry(byte[] headerBuf) {
|
||||
this();
|
||||
this.parseTarHeader(headerBuf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor to create an entry from an existing TarHeader object.
|
||||
*
|
||||
* This method is useful to add new entries programmatically (e.g. for
|
||||
* adding files or directories that do not exist in the file system).
|
||||
*
|
||||
* @param header
|
||||
*
|
||||
*/
|
||||
public TarEntry(TarHeader header) {
|
||||
this.file = null;
|
||||
this.header = header;
|
||||
}
|
||||
|
||||
public boolean equals(TarEntry it) {
|
||||
return header.name.toString().equals(it.header.name.toString());
|
||||
}
|
||||
|
||||
public boolean isDescendent(TarEntry desc) {
|
||||
return desc.header.name.toString().startsWith(header.name.toString());
|
||||
}
|
||||
|
||||
public TarHeader getHeader() {
|
||||
return header;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
String name = header.name.toString();
|
||||
if (header.namePrefix != null && !header.namePrefix.toString().equals("")) {
|
||||
name = header.namePrefix.toString() + "/" + name;
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
header.name = new StringBuffer(name);
|
||||
}
|
||||
|
||||
public int getUserId() {
|
||||
return header.userId;
|
||||
}
|
||||
|
||||
public void setUserId(int userId) {
|
||||
header.userId = userId;
|
||||
}
|
||||
|
||||
public int getGroupId() {
|
||||
return header.groupId;
|
||||
}
|
||||
|
||||
public void setGroupId(int groupId) {
|
||||
header.groupId = groupId;
|
||||
}
|
||||
|
||||
public String getUserName() {
|
||||
return header.userName.toString();
|
||||
}
|
||||
|
||||
public void setUserName(String userName) {
|
||||
header.userName = new StringBuffer(userName);
|
||||
}
|
||||
|
||||
public String getGroupName() {
|
||||
return header.groupName.toString();
|
||||
}
|
||||
|
||||
public void setGroupName(String groupName) {
|
||||
header.groupName = new StringBuffer(groupName);
|
||||
}
|
||||
|
||||
public void setIds(int userId, int groupId) {
|
||||
this.setUserId(userId);
|
||||
this.setGroupId(groupId);
|
||||
}
|
||||
|
||||
public void setModTime(long time) {
|
||||
header.modTime = time / 1000;
|
||||
}
|
||||
|
||||
public void setModTime(Date time) {
|
||||
header.modTime = time.getTime() / 1000;
|
||||
}
|
||||
|
||||
public Date getModTime() {
|
||||
return new Date(header.modTime * 1000);
|
||||
}
|
||||
|
||||
public File getFile() {
|
||||
return this.file;
|
||||
}
|
||||
|
||||
public long getSize() {
|
||||
return header.size;
|
||||
}
|
||||
|
||||
public void setSize(long size) {
|
||||
header.size = size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the org.kamrazafar.jtar entry is a directory
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public boolean isDirectory() {
|
||||
if (this.file != null)
|
||||
return this.file.isDirectory();
|
||||
|
||||
if (header != null) {
|
||||
if (header.linkFlag == TarHeader.LF_DIR)
|
||||
return true;
|
||||
|
||||
if (header.name.toString().endsWith("/"))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract header from File
|
||||
*
|
||||
* @param entryName
|
||||
*/
|
||||
public void extractTarHeader(String entryName) {
|
||||
header = TarHeader.createHeader(entryName, file.length(), file.lastModified() / 1000, file.isDirectory());
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate checksum
|
||||
*
|
||||
* @param buf
|
||||
* @return
|
||||
*/
|
||||
public long computeCheckSum(byte[] buf) {
|
||||
long sum = 0;
|
||||
|
||||
for (int i = 0; i < buf.length; ++i) {
|
||||
sum += 255 & buf[i];
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the header to the byte buffer
|
||||
*
|
||||
* @param outbuf
|
||||
*/
|
||||
public void writeEntryHeader(byte[] outbuf) {
|
||||
int offset = 0;
|
||||
|
||||
offset = TarHeader.getNameBytes(header.name, outbuf, offset, TarHeader.NAMELEN);
|
||||
offset = Octal.getOctalBytes(header.mode, outbuf, offset, TarHeader.MODELEN);
|
||||
offset = Octal.getOctalBytes(header.userId, outbuf, offset, TarHeader.UIDLEN);
|
||||
offset = Octal.getOctalBytes(header.groupId, outbuf, offset, TarHeader.GIDLEN);
|
||||
|
||||
long size = header.size;
|
||||
|
||||
offset = Octal.getLongOctalBytes(size, outbuf, offset, TarHeader.SIZELEN);
|
||||
offset = Octal.getLongOctalBytes(header.modTime, outbuf, offset, TarHeader.MODTIMELEN);
|
||||
|
||||
int csOffset = offset;
|
||||
for (int c = 0; c < TarHeader.CHKSUMLEN; ++c)
|
||||
outbuf[offset++] = (byte) ' ';
|
||||
|
||||
outbuf[offset++] = header.linkFlag;
|
||||
|
||||
offset = TarHeader.getNameBytes(header.linkName, outbuf, offset, TarHeader.NAMELEN);
|
||||
offset = TarHeader.getNameBytes(header.magic, outbuf, offset, TarHeader.USTAR_MAGICLEN);
|
||||
offset = TarHeader.getNameBytes(header.userName, outbuf, offset, TarHeader.USTAR_USER_NAMELEN);
|
||||
offset = TarHeader.getNameBytes(header.groupName, outbuf, offset, TarHeader.USTAR_GROUP_NAMELEN);
|
||||
offset = Octal.getOctalBytes(header.devMajor, outbuf, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset = Octal.getOctalBytes(header.devMinor, outbuf, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset = TarHeader.getNameBytes(header.namePrefix, outbuf, offset, TarHeader.USTAR_FILENAME_PREFIX);
|
||||
|
||||
for (; offset < outbuf.length;)
|
||||
outbuf[offset++] = 0;
|
||||
|
||||
long checkSum = this.computeCheckSum(outbuf);
|
||||
|
||||
Octal.getCheckSumOctalBytes(checkSum, outbuf, csOffset, TarHeader.CHKSUMLEN);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the tar header to the byte buffer
|
||||
*
|
||||
* @param header
|
||||
* @param bh
|
||||
*/
|
||||
public void parseTarHeader(byte[] bh) {
|
||||
int offset = 0;
|
||||
|
||||
header.name = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
|
||||
offset += TarHeader.NAMELEN;
|
||||
|
||||
header.mode = (int) Octal.parseOctal(bh, offset, TarHeader.MODELEN);
|
||||
offset += TarHeader.MODELEN;
|
||||
|
||||
header.userId = (int) Octal.parseOctal(bh, offset, TarHeader.UIDLEN);
|
||||
offset += TarHeader.UIDLEN;
|
||||
|
||||
header.groupId = (int) Octal.parseOctal(bh, offset, TarHeader.GIDLEN);
|
||||
offset += TarHeader.GIDLEN;
|
||||
|
||||
header.size = Octal.parseOctal(bh, offset, TarHeader.SIZELEN);
|
||||
offset += TarHeader.SIZELEN;
|
||||
|
||||
header.modTime = Octal.parseOctal(bh, offset, TarHeader.MODTIMELEN);
|
||||
offset += TarHeader.MODTIMELEN;
|
||||
|
||||
header.checkSum = (int) Octal.parseOctal(bh, offset, TarHeader.CHKSUMLEN);
|
||||
offset += TarHeader.CHKSUMLEN;
|
||||
|
||||
header.linkFlag = bh[offset++];
|
||||
|
||||
header.linkName = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
|
||||
offset += TarHeader.NAMELEN;
|
||||
|
||||
header.magic = TarHeader.parseName(bh, offset, TarHeader.USTAR_MAGICLEN);
|
||||
offset += TarHeader.USTAR_MAGICLEN;
|
||||
|
||||
header.userName = TarHeader.parseName(bh, offset, TarHeader.USTAR_USER_NAMELEN);
|
||||
offset += TarHeader.USTAR_USER_NAMELEN;
|
||||
|
||||
header.groupName = TarHeader.parseName(bh, offset, TarHeader.USTAR_GROUP_NAMELEN);
|
||||
offset += TarHeader.USTAR_GROUP_NAMELEN;
|
||||
|
||||
header.devMajor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset += TarHeader.USTAR_DEVLEN;
|
||||
|
||||
header.devMinor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset += TarHeader.USTAR_DEVLEN;
|
||||
|
||||
header.namePrefix = TarHeader.parseName(bh, offset, TarHeader.USTAR_FILENAME_PREFIX);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* Header
|
||||
*
|
||||
* <pre>
|
||||
* Offset Size Field
|
||||
* 0 100 File name
|
||||
* 100 8 File mode
|
||||
* 108 8 Owner's numeric user ID
|
||||
* 116 8 Group's numeric user ID
|
||||
* 124 12 File size in bytes
|
||||
* 136 12 Last modification time in numeric Unix time format
|
||||
* 148 8 Checksum for header block
|
||||
* 156 1 Link indicator (file type)
|
||||
* 157 100 Name of linked file
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
* File Types
|
||||
*
|
||||
* <pre>
|
||||
* Value Meaning
|
||||
* '0' Normal file
|
||||
* (ASCII NUL) Normal file (now obsolete)
|
||||
* '1' Hard link
|
||||
* '2' Symbolic link
|
||||
* '3' Character special
|
||||
* '4' Block special
|
||||
* '5' Directory
|
||||
* '6' FIFO
|
||||
* '7' Contigous
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
*
|
||||
* Ustar header
|
||||
*
|
||||
* <pre>
|
||||
* Offset Size Field
|
||||
* 257 6 UStar indicator "ustar"
|
||||
* 263 2 UStar version "00"
|
||||
* 265 32 Owner user name
|
||||
* 297 32 Owner group name
|
||||
* 329 8 Device major number
|
||||
* 337 8 Device minor number
|
||||
* 345 155 Filename prefix
|
||||
* </pre>
|
||||
*/
|
||||
|
||||
public class TarHeader {
|
||||
|
||||
/*
|
||||
* Header
|
||||
*/
|
||||
public static final int NAMELEN = 100;
|
||||
public static final int MODELEN = 8;
|
||||
public static final int UIDLEN = 8;
|
||||
public static final int GIDLEN = 8;
|
||||
public static final int SIZELEN = 12;
|
||||
public static final int MODTIMELEN = 12;
|
||||
public static final int CHKSUMLEN = 8;
|
||||
public static final byte LF_OLDNORM = 0;
|
||||
|
||||
/*
|
||||
* File Types
|
||||
*/
|
||||
public static final byte LF_NORMAL = (byte) '0';
|
||||
public static final byte LF_LINK = (byte) '1';
|
||||
public static final byte LF_SYMLINK = (byte) '2';
|
||||
public static final byte LF_CHR = (byte) '3';
|
||||
public static final byte LF_BLK = (byte) '4';
|
||||
public static final byte LF_DIR = (byte) '5';
|
||||
public static final byte LF_FIFO = (byte) '6';
|
||||
public static final byte LF_CONTIG = (byte) '7';
|
||||
|
||||
/*
|
||||
* Ustar header
|
||||
*/
|
||||
|
||||
public static final String USTAR_MAGIC = "ustar"; // POSIX
|
||||
|
||||
public static final int USTAR_MAGICLEN = 8;
|
||||
public static final int USTAR_USER_NAMELEN = 32;
|
||||
public static final int USTAR_GROUP_NAMELEN = 32;
|
||||
public static final int USTAR_DEVLEN = 8;
|
||||
public static final int USTAR_FILENAME_PREFIX = 155;
|
||||
|
||||
// Header values
|
||||
public StringBuffer name;
|
||||
public int mode;
|
||||
public int userId;
|
||||
public int groupId;
|
||||
public long size;
|
||||
public long modTime;
|
||||
public int checkSum;
|
||||
public byte linkFlag;
|
||||
public StringBuffer linkName;
|
||||
public StringBuffer magic; // ustar indicator and version
|
||||
public StringBuffer userName;
|
||||
public StringBuffer groupName;
|
||||
public int devMajor;
|
||||
public int devMinor;
|
||||
public StringBuffer namePrefix;
|
||||
|
||||
public TarHeader() {
|
||||
this.magic = new StringBuffer(TarHeader.USTAR_MAGIC);
|
||||
|
||||
this.name = new StringBuffer();
|
||||
this.linkName = new StringBuffer();
|
||||
|
||||
String user = System.getProperty("user.name", "");
|
||||
|
||||
if (user.length() > 31)
|
||||
user = user.substring(0, 31);
|
||||
|
||||
this.userId = 0;
|
||||
this.groupId = 0;
|
||||
this.userName = new StringBuffer(user);
|
||||
this.groupName = new StringBuffer("");
|
||||
this.namePrefix = new StringBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an entry name from a header buffer.
|
||||
*
|
||||
* @param name
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The header's entry name.
|
||||
*/
|
||||
public static StringBuffer parseName(byte[] header, int offset, int length) {
|
||||
StringBuffer result = new StringBuffer(length);
|
||||
|
||||
int end = offset + length;
|
||||
for (int i = offset; i < end; ++i) {
|
||||
if (header[i] == 0)
|
||||
break;
|
||||
result.append((char) header[i]);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the number of bytes in an entry name.
|
||||
*
|
||||
* @param name
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The number of bytes in a header's entry name.
|
||||
*/
|
||||
public static int getNameBytes(StringBuffer name, byte[] buf, int offset, int length) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < length && i < name.length(); ++i) {
|
||||
buf[offset + i] = (byte) name.charAt(i);
|
||||
}
|
||||
|
||||
for (; i < length; ++i) {
|
||||
buf[offset + i] = 0;
|
||||
}
|
||||
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new header for a file/directory entry.
|
||||
*
|
||||
*
|
||||
* @param name
|
||||
* File name
|
||||
* @param size
|
||||
* File size in bytes
|
||||
* @param modTime
|
||||
* Last modification time in numeric Unix time format
|
||||
* @param dir
|
||||
* Is directory
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public static TarHeader createHeader(String entryName, long size, long modTime, boolean dir) {
|
||||
String name = entryName;
|
||||
name = TarUtils.trim(name.replace(File.separatorChar, '/'), '/');
|
||||
|
||||
TarHeader header = new TarHeader();
|
||||
header.linkName = new StringBuffer("");
|
||||
|
||||
if (name.length() > 100) {
|
||||
header.namePrefix = new StringBuffer(name.substring(0, name.lastIndexOf('/')));
|
||||
header.name = new StringBuffer(name.substring(name.lastIndexOf('/') + 1));
|
||||
} else {
|
||||
header.name = new StringBuffer(name);
|
||||
}
|
||||
|
||||
if (dir) {
|
||||
header.mode = 040755;
|
||||
header.linkFlag = TarHeader.LF_DIR;
|
||||
if (header.name.charAt(header.name.length() - 1) != '/') {
|
||||
header.name.append("/");
|
||||
}
|
||||
header.size = 0;
|
||||
} else {
|
||||
header.mode = 0100644;
|
||||
header.linkFlag = TarHeader.LF_NORMAL;
|
||||
header.size = size;
|
||||
}
|
||||
|
||||
header.modTime = modTime;
|
||||
header.checkSum = 0;
|
||||
header.devMajor = 0;
|
||||
header.devMinor = 0;
|
||||
|
||||
return header;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarInputStream extends FilterInputStream {
|
||||
|
||||
private static final int SKIP_BUFFER_SIZE = 2048;
|
||||
private TarEntry currentEntry;
|
||||
private long currentFileSize;
|
||||
private long bytesRead;
|
||||
private boolean defaultSkip = false;
|
||||
|
||||
public TarInputStream(InputStream in) {
|
||||
super(in);
|
||||
currentFileSize = 0;
|
||||
bytesRead = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markSupported() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public synchronized void mark(int readlimit) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public synchronized void reset() throws IOException {
|
||||
throw new IOException("mark/reset not supported");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a byte
|
||||
*
|
||||
* @see java.io.FilterInputStream#read()
|
||||
*/
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
byte[] buf = new byte[1];
|
||||
|
||||
int res = this.read(buf, 0, 1);
|
||||
|
||||
if (res != -1) {
|
||||
return 0xFF & buf[0];
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the bytes being read exceed the entry size and adjusts the byte
|
||||
* array length. Updates the byte counters
|
||||
*
|
||||
*
|
||||
* @see java.io.FilterInputStream#read(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public int read(byte[] b, int off, int len) throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentFileSize == currentEntry.getSize()) {
|
||||
return -1;
|
||||
} else if ((currentEntry.getSize() - currentFileSize) < len) {
|
||||
len = (int) (currentEntry.getSize() - currentFileSize);
|
||||
}
|
||||
}
|
||||
|
||||
int br = super.read(b, off, len);
|
||||
|
||||
if (br != -1) {
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += br;
|
||||
}
|
||||
|
||||
bytesRead += br;
|
||||
}
|
||||
|
||||
return br;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the next entry in the tar file
|
||||
*
|
||||
* @return TarEntry
|
||||
* @throws IOException
|
||||
*/
|
||||
public TarEntry getNextEntry() throws IOException {
|
||||
closeCurrentEntry();
|
||||
|
||||
byte[] header = new byte[TarConstants.HEADER_BLOCK];
|
||||
byte[] theader = new byte[TarConstants.HEADER_BLOCK];
|
||||
int tr = 0;
|
||||
|
||||
// Read full header
|
||||
while (tr < TarConstants.HEADER_BLOCK) {
|
||||
int res = read(theader, 0, TarConstants.HEADER_BLOCK - tr);
|
||||
|
||||
if (res < 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
System.arraycopy(theader, 0, header, tr, res);
|
||||
tr += res;
|
||||
}
|
||||
|
||||
// Check if record is null
|
||||
boolean eof = true;
|
||||
for (byte b : header) {
|
||||
if (b != 0) {
|
||||
eof = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!eof) {
|
||||
currentEntry = new TarEntry(header);
|
||||
}
|
||||
|
||||
return currentEntry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current offset (in bytes) from the beginning of the stream.
|
||||
* This can be used to find out at which point in a tar file an entry's content begins, for instance.
|
||||
*/
|
||||
public long getCurrentOffset() {
|
||||
return bytesRead;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the current tar entry
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void closeCurrentEntry() throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentEntry.getSize() > currentFileSize) {
|
||||
// Not fully read, skip rest of the bytes
|
||||
long bs = 0;
|
||||
while (bs < currentEntry.getSize() - currentFileSize) {
|
||||
long res = skip(currentEntry.getSize() - currentFileSize - bs);
|
||||
|
||||
if (res == 0 && currentEntry.getSize() - currentFileSize > 0) {
|
||||
// I suspect file corruption
|
||||
throw new IOException("Possible tar file corruption");
|
||||
}
|
||||
|
||||
bs += res;
|
||||
}
|
||||
}
|
||||
|
||||
currentEntry = null;
|
||||
currentFileSize = 0L;
|
||||
skipPad();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skips the pad at the end of each tar entry file content
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void skipPad() throws IOException {
|
||||
if (bytesRead > 0) {
|
||||
int extra = (int) (bytesRead % TarConstants.DATA_BLOCK);
|
||||
|
||||
if (extra > 0) {
|
||||
long bs = 0;
|
||||
while (bs < TarConstants.DATA_BLOCK - extra) {
|
||||
long res = skip(TarConstants.DATA_BLOCK - extra - bs);
|
||||
bs += res;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skips 'n' bytes on the InputStream<br>
|
||||
* Overrides default implementation of skip
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public long skip(long n) throws IOException {
|
||||
if (defaultSkip) {
|
||||
// use skip method of parent stream
|
||||
// may not work if skip not implemented by parent
|
||||
long bs = super.skip(n);
|
||||
bytesRead += bs;
|
||||
|
||||
return bs;
|
||||
}
|
||||
|
||||
if (n <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
long left = n;
|
||||
byte[] sBuff = new byte[SKIP_BUFFER_SIZE];
|
||||
|
||||
while (left > 0) {
|
||||
int res = read(sBuff, 0, (int) (left < SKIP_BUFFER_SIZE ? left : SKIP_BUFFER_SIZE));
|
||||
if (res < 0) {
|
||||
break;
|
||||
}
|
||||
left -= res;
|
||||
}
|
||||
|
||||
return n - left;
|
||||
}
|
||||
|
||||
public boolean isDefaultSkip() {
|
||||
return defaultSkip;
|
||||
}
|
||||
|
||||
public void setDefaultSkip(boolean defaultSkip) {
|
||||
this.defaultSkip = defaultSkip;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarOutputStream extends OutputStream {
|
||||
private final OutputStream out;
|
||||
private long bytesWritten;
|
||||
private long currentFileSize;
|
||||
private TarEntry currentEntry;
|
||||
|
||||
public TarOutputStream(OutputStream out) {
|
||||
this.out = out;
|
||||
bytesWritten = 0;
|
||||
currentFileSize = 0;
|
||||
}
|
||||
|
||||
public TarOutputStream(final File fout) throws FileNotFoundException {
|
||||
this.out = new BufferedOutputStream(new FileOutputStream(fout));
|
||||
bytesWritten = 0;
|
||||
currentFileSize = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a file for writing.
|
||||
*/
|
||||
public TarOutputStream(final File fout, final boolean append) throws IOException {
|
||||
@SuppressWarnings("resource")
|
||||
RandomAccessFile raf = new RandomAccessFile(fout, "rw");
|
||||
final long fileSize = fout.length();
|
||||
if (append && fileSize > TarConstants.EOF_BLOCK) {
|
||||
raf.seek(fileSize - TarConstants.EOF_BLOCK);
|
||||
}
|
||||
out = new BufferedOutputStream(new FileOutputStream(raf.getFD()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends the EOF record and closes the stream
|
||||
*
|
||||
* @see java.io.FilterOutputStream#close()
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
closeCurrentEntry();
|
||||
write( new byte[TarConstants.EOF_BLOCK] );
|
||||
out.close();
|
||||
}
|
||||
/**
|
||||
* Writes a byte to the stream and updates byte counters
|
||||
*
|
||||
* @see java.io.FilterOutputStream#write(int)
|
||||
*/
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
out.write( b );
|
||||
bytesWritten += 1;
|
||||
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the bytes being written exceed the current entry size.
|
||||
*
|
||||
* @see java.io.FilterOutputStream#write(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException {
|
||||
if (currentEntry != null && !currentEntry.isDirectory()) {
|
||||
if (currentEntry.getSize() < currentFileSize + len) {
|
||||
throw new IOException( "The current entry[" + currentEntry.getName() + "] size["
|
||||
+ currentEntry.getSize() + "] is smaller than the bytes[" + ( currentFileSize + len )
|
||||
+ "] being written." );
|
||||
}
|
||||
}
|
||||
|
||||
out.write( b, off, len );
|
||||
|
||||
bytesWritten += len;
|
||||
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += len;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the next tar entry header on the stream
|
||||
*
|
||||
* @param entry
|
||||
* @throws IOException
|
||||
*/
|
||||
public void putNextEntry(TarEntry entry) throws IOException {
|
||||
closeCurrentEntry();
|
||||
|
||||
byte[] header = new byte[TarConstants.HEADER_BLOCK];
|
||||
entry.writeEntryHeader( header );
|
||||
|
||||
write( header );
|
||||
|
||||
currentEntry = entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the current tar entry
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void closeCurrentEntry() throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentEntry.getSize() > currentFileSize) {
|
||||
throw new IOException( "The current entry[" + currentEntry.getName() + "] of size["
|
||||
+ currentEntry.getSize() + "] has not been fully written." );
|
||||
}
|
||||
|
||||
currentEntry = null;
|
||||
currentFileSize = 0;
|
||||
|
||||
pad();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pads the last content block
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void pad() throws IOException {
|
||||
if (bytesWritten > 0) {
|
||||
int extra = (int) ( bytesWritten % TarConstants.DATA_BLOCK );
|
||||
|
||||
if (extra > 0) {
|
||||
write( new byte[TarConstants.DATA_BLOCK - extra] );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* @author Kamran
|
||||
*
|
||||
*/
|
||||
public class TarUtils {
|
||||
/**
|
||||
* Determines the tar file size of the given folder/file path
|
||||
*
|
||||
* @param path
|
||||
* @return
|
||||
*/
|
||||
public static long calculateTarSize(File path) {
|
||||
return tarSize(path) + TarConstants.EOF_BLOCK;
|
||||
}
|
||||
|
||||
private static long tarSize(File dir) {
|
||||
long size = 0;
|
||||
|
||||
if (dir.isFile()) {
|
||||
return entrySize(dir.length());
|
||||
} else {
|
||||
File[] subFiles = dir.listFiles();
|
||||
|
||||
if (subFiles != null && subFiles.length > 0) {
|
||||
for (File file : subFiles) {
|
||||
if (file.isFile()) {
|
||||
size += entrySize(file.length());
|
||||
} else {
|
||||
size += tarSize(file);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Empty folder header
|
||||
return TarConstants.HEADER_BLOCK;
|
||||
}
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
private static long entrySize(long fileSize) {
|
||||
long size = 0;
|
||||
size += TarConstants.HEADER_BLOCK; // Header
|
||||
size += fileSize; // File size
|
||||
|
||||
long extra = size % TarConstants.DATA_BLOCK;
|
||||
|
||||
if (extra > 0) {
|
||||
size += (TarConstants.DATA_BLOCK - extra); // pad
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
public static String trim(String s, char c) {
|
||||
StringBuffer tmp = new StringBuffer(s);
|
||||
for (int i = 0; i < tmp.length(); i++) {
|
||||
if (tmp.charAt(i) != c) {
|
||||
break;
|
||||
} else {
|
||||
tmp.deleteCharAt(i);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = tmp.length() - 1; i >= 0; i--) {
|
||||
if (tmp.charAt(i) != c) {
|
||||
break;
|
||||
} else {
|
||||
tmp.deleteCharAt(i);
|
||||
}
|
||||
}
|
||||
|
||||
return tmp.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
|
||||
package org.kivy.android;
|
||||
|
||||
import java.net.Socket;
|
||||
import java.net.InetSocketAddress;
|
||||
|
||||
import android.os.SystemClock;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import android.app.*;
|
||||
import android.content.*;
|
||||
import android.view.*;
|
||||
import android.view.SurfaceView;
|
||||
import android.app.Activity;
|
||||
import android.content.Intent;
|
||||
import android.util.Log;
|
||||
import android.widget.Toast;
|
||||
import android.os.AsyncTask;
|
||||
import android.os.Bundle;
|
||||
import android.os.PowerManager;
|
||||
import android.graphics.PixelFormat;
|
||||
import android.view.SurfaceHolder;
|
||||
import android.content.Context;
|
||||
import android.content.pm.ActivityInfo;
|
||||
import android.content.pm.PackageManager;
|
||||
import android.content.pm.ApplicationInfo;
|
||||
import android.content.Intent;
|
||||
import android.widget.ImageView;
|
||||
import java.io.InputStream;
|
||||
import android.graphics.Bitmap;
|
||||
import android.graphics.BitmapFactory;
|
||||
import android.graphics.Color;
|
||||
|
||||
import android.widget.AbsoluteLayout;
|
||||
|
||||
import android.webkit.WebViewClient;
|
||||
import android.webkit.WebView;
|
||||
|
||||
import org.kivy.android.PythonUtil;
|
||||
|
||||
import org.renpy.android.ResourceManager;
|
||||
import org.renpy.android.AssetExtract;
|
||||
|
||||
public class PythonActivity extends Activity {
|
||||
// This activity is modified from a mixture of the SDLActivity and
|
||||
// PythonActivity in the SDL2 bootstrap, but removing all the SDL2
|
||||
// specifics.
|
||||
|
||||
private static final String TAG = "PythonActivity";
|
||||
|
||||
public static PythonActivity mActivity = null;
|
||||
|
||||
/** If shared libraries (e.g. the native application) could not be loaded. */
|
||||
public static boolean mBrokenLibraries;
|
||||
|
||||
protected static Thread mPythonThread;
|
||||
|
||||
private ResourceManager resourceManager = null;
|
||||
private Bundle mMetaData = null;
|
||||
private PowerManager.WakeLock mWakeLock = null;
|
||||
|
||||
public String getAppRoot() {
|
||||
String app_root = getFilesDir().getAbsolutePath() + "/app";
|
||||
return app_root;
|
||||
}
|
||||
|
||||
public static void initialize() {
|
||||
// The static nature of the singleton and Android quirkiness force us to initialize everything here
|
||||
// Otherwise, when exiting the app and returning to it, these variables *keep* their pre exit values
|
||||
mBrokenLibraries = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onCreate(Bundle savedInstanceState) {
|
||||
Log.v(TAG, "My oncreate running");
|
||||
resourceManager = new ResourceManager(this);
|
||||
|
||||
Log.v(TAG, "Ready to unpack");
|
||||
File app_root_file = new File(getAppRoot());
|
||||
unpackData("private", app_root_file);
|
||||
|
||||
Log.v(TAG, "About to do super onCreate");
|
||||
super.onCreate(savedInstanceState);
|
||||
Log.v(TAG, "Did super onCreate");
|
||||
|
||||
this.mActivity = this;
|
||||
//this.showLoadingScreen();
|
||||
Log.v("Python", "Device: " + android.os.Build.DEVICE);
|
||||
Log.v("Python", "Model: " + android.os.Build.MODEL);
|
||||
|
||||
//Log.v(TAG, "Ready to unpack");
|
||||
//new UnpackFilesTask().execute(getAppRoot());
|
||||
|
||||
PythonActivity.initialize();
|
||||
|
||||
// Load shared libraries
|
||||
String errorMsgBrokenLib = "";
|
||||
try {
|
||||
loadLibraries();
|
||||
} catch(UnsatisfiedLinkError e) {
|
||||
System.err.println(e.getMessage());
|
||||
mBrokenLibraries = true;
|
||||
errorMsgBrokenLib = e.getMessage();
|
||||
} catch(Exception e) {
|
||||
System.err.println(e.getMessage());
|
||||
mBrokenLibraries = true;
|
||||
errorMsgBrokenLib = e.getMessage();
|
||||
}
|
||||
|
||||
if (mBrokenLibraries)
|
||||
{
|
||||
AlertDialog.Builder dlgAlert = new AlertDialog.Builder(this);
|
||||
dlgAlert.setMessage("An error occurred while trying to load the application libraries. Please try again and/or reinstall."
|
||||
+ System.getProperty("line.separator")
|
||||
+ System.getProperty("line.separator")
|
||||
+ "Error: " + errorMsgBrokenLib);
|
||||
dlgAlert.setTitle("Python Error");
|
||||
dlgAlert.setPositiveButton("Exit",
|
||||
new DialogInterface.OnClickListener() {
|
||||
@Override
|
||||
public void onClick(DialogInterface dialog,int id) {
|
||||
// if this button is clicked, close current activity
|
||||
PythonActivity.mActivity.finish();
|
||||
}
|
||||
});
|
||||
dlgAlert.setCancelable(false);
|
||||
dlgAlert.create().show();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Set up the Python environment
|
||||
String app_root_dir = getAppRoot();
|
||||
String mFilesDirectory = mActivity.getFilesDir().getAbsolutePath();
|
||||
|
||||
Log.v(TAG, "Setting env vars for start.c and Python to use");
|
||||
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
|
||||
PythonActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
|
||||
PythonActivity.nativeSetEnv("ANDROID_UNPACK", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("PYTHONHOME", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("PYTHONPATH", app_root_dir + ":" + app_root_dir + "/lib");
|
||||
PythonActivity.nativeSetEnv("PYTHONOPTIMIZE", "2");
|
||||
|
||||
try {
|
||||
Log.v(TAG, "Access to our meta-data...");
|
||||
mActivity.mMetaData = mActivity.getPackageManager().getApplicationInfo(
|
||||
mActivity.getPackageName(), PackageManager.GET_META_DATA).metaData;
|
||||
|
||||
PowerManager pm = (PowerManager) mActivity.getSystemService(Context.POWER_SERVICE);
|
||||
if ( mActivity.mMetaData.getInt("wakelock") == 1 ) {
|
||||
mActivity.mWakeLock = pm.newWakeLock(PowerManager.SCREEN_BRIGHT_WAKE_LOCK, "Screen On");
|
||||
mActivity.mWakeLock.acquire();
|
||||
}
|
||||
} catch (PackageManager.NameNotFoundException e) {
|
||||
}
|
||||
|
||||
final Thread pythonThread = new Thread(new PythonMain(), "PythonThread");
|
||||
PythonActivity.mPythonThread = pythonThread;
|
||||
pythonThread.start();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onDestroy() {
|
||||
Log.i("Destroy", "end of app");
|
||||
super.onDestroy();
|
||||
|
||||
// make sure all child threads (python_thread) are stopped
|
||||
android.os.Process.killProcess(android.os.Process.myPid());
|
||||
}
|
||||
|
||||
public void loadLibraries() {
|
||||
String app_root = new String(getAppRoot());
|
||||
File app_root_file = new File(app_root);
|
||||
PythonUtil.loadLibraries(app_root_file);
|
||||
}
|
||||
|
||||
public void recursiveDelete(File f) {
|
||||
if (f.isDirectory()) {
|
||||
for (File r : f.listFiles()) {
|
||||
recursiveDelete(r);
|
||||
}
|
||||
}
|
||||
f.delete();
|
||||
}
|
||||
|
||||
/**
|
||||
* Show an error using a toast. (Only makes sense from non-UI
|
||||
* threads.)
|
||||
*/
|
||||
public void toastError(final String msg) {
|
||||
|
||||
final Activity thisActivity = this;
|
||||
|
||||
runOnUiThread(new Runnable () {
|
||||
public void run() {
|
||||
Toast.makeText(thisActivity, msg, Toast.LENGTH_LONG).show();
|
||||
}
|
||||
});
|
||||
|
||||
// Wait to show the error.
|
||||
synchronized (this) {
|
||||
try {
|
||||
this.wait(1000);
|
||||
} catch (InterruptedException e) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void unpackData(final String resource, File target) {
|
||||
|
||||
Log.v(TAG, "UNPACKING!!! " + resource + " " + target.getName());
|
||||
|
||||
// The version of data in memory and on disk.
|
||||
String data_version = resourceManager.getString(resource + "_version");
|
||||
String disk_version = null;
|
||||
|
||||
Log.v(TAG, "Data version is " + data_version);
|
||||
|
||||
// If no version, no unpacking is necessary.
|
||||
if (data_version == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check the current disk version, if any.
|
||||
String filesDir = target.getAbsolutePath();
|
||||
String disk_version_fn = filesDir + "/" + resource + ".version";
|
||||
|
||||
try {
|
||||
byte buf[] = new byte[64];
|
||||
InputStream is = new FileInputStream(disk_version_fn);
|
||||
int len = is.read(buf);
|
||||
disk_version = new String(buf, 0, len);
|
||||
is.close();
|
||||
} catch (Exception e) {
|
||||
disk_version = "";
|
||||
}
|
||||
|
||||
// If the disk data is out of date, extract it and write the
|
||||
// version file.
|
||||
// if (! data_version.equals(disk_version)) {
|
||||
if (! data_version.equals(disk_version)) {
|
||||
Log.v(TAG, "Extracting " + resource + " assets.");
|
||||
|
||||
recursiveDelete(target);
|
||||
target.mkdirs();
|
||||
|
||||
AssetExtract ae = new AssetExtract(this);
|
||||
if (!ae.extractTar(resource + ".mp3", target.getAbsolutePath())) {
|
||||
toastError("Could not extract " + resource + " data.");
|
||||
}
|
||||
|
||||
try {
|
||||
// Write .nomedia.
|
||||
new File(target, ".nomedia").createNewFile();
|
||||
|
||||
// Write version file.
|
||||
FileOutputStream os = new FileOutputStream(disk_version_fn);
|
||||
os.write(data_version.getBytes());
|
||||
os.close();
|
||||
} catch (Exception e) {
|
||||
Log.w("python", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long lastBackClick = SystemClock.elapsedRealtime();
|
||||
@Override
|
||||
public boolean onKeyDown(int keyCode, KeyEvent event) {
|
||||
// If it wasn't the Back key or there's no web page history, bubble up to the default
|
||||
// system behavior (probably exit the activity)
|
||||
if (SystemClock.elapsedRealtime() - lastBackClick > 2000){
|
||||
lastBackClick = SystemClock.elapsedRealtime();
|
||||
Toast.makeText(this, "Click again to close the app",
|
||||
Toast.LENGTH_LONG).show();
|
||||
return true;
|
||||
}
|
||||
|
||||
lastBackClick = SystemClock.elapsedRealtime();
|
||||
return super.onKeyDown(keyCode, event);
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
// Listener interface for onNewIntent
|
||||
//
|
||||
|
||||
public interface NewIntentListener {
|
||||
void onNewIntent(Intent intent);
|
||||
}
|
||||
|
||||
private List<NewIntentListener> newIntentListeners = null;
|
||||
|
||||
public void registerNewIntentListener(NewIntentListener listener) {
|
||||
if ( this.newIntentListeners == null )
|
||||
this.newIntentListeners = Collections.synchronizedList(new ArrayList<NewIntentListener>());
|
||||
this.newIntentListeners.add(listener);
|
||||
}
|
||||
|
||||
public void unregisterNewIntentListener(NewIntentListener listener) {
|
||||
if ( this.newIntentListeners == null )
|
||||
return;
|
||||
this.newIntentListeners.remove(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onNewIntent(Intent intent) {
|
||||
if ( this.newIntentListeners == null )
|
||||
return;
|
||||
this.onResume();
|
||||
synchronized ( this.newIntentListeners ) {
|
||||
Iterator<NewIntentListener> iterator = this.newIntentListeners.iterator();
|
||||
while ( iterator.hasNext() ) {
|
||||
(iterator.next()).onNewIntent(intent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
// Listener interface for onActivityResult
|
||||
//
|
||||
|
||||
public interface ActivityResultListener {
|
||||
void onActivityResult(int requestCode, int resultCode, Intent data);
|
||||
}
|
||||
|
||||
private List<ActivityResultListener> activityResultListeners = null;
|
||||
|
||||
public void registerActivityResultListener(ActivityResultListener listener) {
|
||||
if ( this.activityResultListeners == null )
|
||||
this.activityResultListeners = Collections.synchronizedList(new ArrayList<ActivityResultListener>());
|
||||
this.activityResultListeners.add(listener);
|
||||
}
|
||||
|
||||
public void unregisterActivityResultListener(ActivityResultListener listener) {
|
||||
if ( this.activityResultListeners == null )
|
||||
return;
|
||||
this.activityResultListeners.remove(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onActivityResult(int requestCode, int resultCode, Intent intent) {
|
||||
if ( this.activityResultListeners == null )
|
||||
return;
|
||||
this.onResume();
|
||||
synchronized ( this.activityResultListeners ) {
|
||||
Iterator<ActivityResultListener> iterator = this.activityResultListeners.iterator();
|
||||
while ( iterator.hasNext() )
|
||||
(iterator.next()).onActivityResult(requestCode, resultCode, intent);
|
||||
}
|
||||
}
|
||||
|
||||
public static void start_service(String serviceTitle, String serviceDescription,
|
||||
String pythonServiceArgument) {
|
||||
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
|
||||
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
|
||||
String filesDirectory = argument;
|
||||
String app_root_dir = PythonActivity.mActivity.getAppRoot();
|
||||
serviceIntent.putExtra("androidPrivate", argument);
|
||||
serviceIntent.putExtra("androidArgument", app_root_dir);
|
||||
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
|
||||
serviceIntent.putExtra("pythonName", "python");
|
||||
serviceIntent.putExtra("pythonHome", app_root_dir);
|
||||
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
|
||||
serviceIntent.putExtra("serviceTitle", serviceTitle);
|
||||
serviceIntent.putExtra("serviceDescription", serviceDescription);
|
||||
serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument);
|
||||
PythonActivity.mActivity.startService(serviceIntent);
|
||||
}
|
||||
|
||||
public static void stop_service() {
|
||||
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
|
||||
PythonActivity.mActivity.stopService(serviceIntent);
|
||||
}
|
||||
|
||||
|
||||
public static native void nativeSetEnv(String j_name, String j_value);
|
||||
public static native int nativeInit(Object arguments);
|
||||
|
||||
}
|
||||
|
||||
|
||||
class PythonMain implements Runnable {
|
||||
@Override
|
||||
public void run() {
|
||||
PythonActivity.nativeInit(new String[0]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package org.kivy.android.concurrency;
|
||||
|
||||
import java.util.concurrent.locks.Condition;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Created by ryan on 3/28/14.
|
||||
*/
|
||||
public class PythonEvent {
|
||||
private final Lock lock = new ReentrantLock();
|
||||
private final Condition cond = lock.newCondition();
|
||||
private boolean flag = false;
|
||||
|
||||
public void set() {
|
||||
lock.lock();
|
||||
try {
|
||||
flag = true;
|
||||
cond.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void wait_() throws InterruptedException {
|
||||
lock.lock();
|
||||
try {
|
||||
while (!flag) {
|
||||
cond.await();
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
lock.lock();
|
||||
try {
|
||||
flag = false;
|
||||
cond.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package org.kivy.android.concurrency;
|
||||
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Created by ryan on 3/28/14.
|
||||
*/
|
||||
public class PythonLock {
|
||||
private final Lock lock = new ReentrantLock();
|
||||
|
||||
public void acquire() {
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package org.renpy.android;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import android.content.BroadcastReceiver;
|
||||
import android.content.Context;
|
||||
import android.content.Intent;
|
||||
import android.content.IntentFilter;
|
||||
import android.hardware.Sensor;
|
||||
import android.hardware.SensorEvent;
|
||||
import android.hardware.SensorEventListener;
|
||||
import android.hardware.SensorManager;
|
||||
import android.net.ConnectivityManager;
|
||||
import android.net.NetworkInfo;
|
||||
import android.net.wifi.ScanResult;
|
||||
import android.net.wifi.WifiManager;
|
||||
import android.os.Vibrator;
|
||||
import android.view.View;
|
||||
|
||||
/**
|
||||
* Methods that are expected to be called via JNI, to access the device's
|
||||
* non-screen hardware. (For example, the vibration and accelerometer.)
|
||||
*/
|
||||
public class Hardware {
|
||||
|
||||
// The context.
|
||||
static Context context;
|
||||
static View view;
|
||||
|
||||
/**
|
||||
* Vibrate for s seconds.
|
||||
*/
|
||||
public static void vibrate(double s) {
|
||||
Vibrator v = (Vibrator) context
|
||||
.getSystemService(Context.VIBRATOR_SERVICE);
|
||||
if (v != null) {
|
||||
v.vibrate((int) (1000 * s));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an Overview of all Hardware Sensors of an Android Device
|
||||
*/
|
||||
public static String getHardwareSensors() {
|
||||
SensorManager sm = (SensorManager) context
|
||||
.getSystemService(Context.SENSOR_SERVICE);
|
||||
List<Sensor> allSensors = sm.getSensorList(Sensor.TYPE_ALL);
|
||||
|
||||
if (allSensors != null) {
|
||||
String resultString = "";
|
||||
for (Sensor s : allSensors) {
|
||||
resultString += String.format("Name=" + s.getName());
|
||||
resultString += String.format(",Vendor=" + s.getVendor());
|
||||
resultString += String.format(",Version=" + s.getVersion());
|
||||
resultString += String.format(",MaximumRange="
|
||||
+ s.getMaximumRange());
|
||||
// XXX MinDelay is not in the 2.2
|
||||
// resultString += String.format(",MinDelay=" +
|
||||
// s.getMinDelay());
|
||||
resultString += String.format(",Power=" + s.getPower());
|
||||
resultString += String.format(",Type=" + s.getType() + "\n");
|
||||
}
|
||||
return resultString;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Access to 3 Axis Hardware Sensors Accelerometer, Orientation and
|
||||
* Magnetic Field Sensors
|
||||
*/
|
||||
public static class generic3AxisSensor implements SensorEventListener {
|
||||
private final SensorManager sSensorManager;
|
||||
private final Sensor sSensor;
|
||||
private final int sSensorType;
|
||||
SensorEvent sSensorEvent;
|
||||
|
||||
public generic3AxisSensor(int sensorType) {
|
||||
sSensorType = sensorType;
|
||||
sSensorManager = (SensorManager) context
|
||||
.getSystemService(Context.SENSOR_SERVICE);
|
||||
sSensor = sSensorManager.getDefaultSensor(sSensorType);
|
||||
}
|
||||
|
||||
public void onAccuracyChanged(Sensor sensor, int accuracy) {
|
||||
}
|
||||
|
||||
public void onSensorChanged(SensorEvent event) {
|
||||
sSensorEvent = event;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable or disable the Sensor by registering/unregistering
|
||||
*/
|
||||
public void changeStatus(boolean enable) {
|
||||
if (enable) {
|
||||
sSensorManager.registerListener(this, sSensor,
|
||||
SensorManager.SENSOR_DELAY_NORMAL);
|
||||
} else {
|
||||
sSensorManager.unregisterListener(this, sSensor);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the Sensor
|
||||
*/
|
||||
public float[] readSensor() {
|
||||
if (sSensorEvent != null) {
|
||||
return sSensorEvent.values;
|
||||
} else {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static generic3AxisSensor accelerometerSensor = null;
|
||||
public static generic3AxisSensor orientationSensor = null;
|
||||
public static generic3AxisSensor magneticFieldSensor = null;
|
||||
|
||||
/**
|
||||
* functions for backward compatibility reasons
|
||||
*/
|
||||
|
||||
public static void accelerometerEnable(boolean enable) {
|
||||
if (accelerometerSensor == null)
|
||||
accelerometerSensor = new generic3AxisSensor(
|
||||
Sensor.TYPE_ACCELEROMETER);
|
||||
accelerometerSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] accelerometerReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (accelerometerSensor == null)
|
||||
return rv;
|
||||
return (float[]) accelerometerSensor.readSensor();
|
||||
}
|
||||
|
||||
public static void orientationSensorEnable(boolean enable) {
|
||||
if (orientationSensor == null)
|
||||
orientationSensor = new generic3AxisSensor(Sensor.TYPE_ORIENTATION);
|
||||
orientationSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] orientationSensorReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (orientationSensor == null)
|
||||
return rv;
|
||||
return (float[]) orientationSensor.readSensor();
|
||||
}
|
||||
|
||||
public static void magneticFieldSensorEnable(boolean enable) {
|
||||
if (magneticFieldSensor == null)
|
||||
magneticFieldSensor = new generic3AxisSensor(
|
||||
Sensor.TYPE_MAGNETIC_FIELD);
|
||||
magneticFieldSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] magneticFieldSensorReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (magneticFieldSensor == null)
|
||||
return rv;
|
||||
return (float[]) magneticFieldSensor.readSensor();
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan WiFi networks
|
||||
*/
|
||||
static List<ScanResult> latestResult;
|
||||
|
||||
public static void enableWifiScanner() {
|
||||
IntentFilter i = new IntentFilter();
|
||||
i.addAction(WifiManager.SCAN_RESULTS_AVAILABLE_ACTION);
|
||||
|
||||
context.registerReceiver(new BroadcastReceiver() {
|
||||
|
||||
@Override
|
||||
public void onReceive(Context c, Intent i) {
|
||||
// Code to execute when SCAN_RESULTS_AVAILABLE_ACTION event
|
||||
// occurs
|
||||
WifiManager w = (WifiManager) c
|
||||
.getSystemService(Context.WIFI_SERVICE);
|
||||
latestResult = w.getScanResults(); // Returns a <list> of
|
||||
// scanResults
|
||||
}
|
||||
|
||||
}, i);
|
||||
|
||||
}
|
||||
|
||||
public static String scanWifi() {
|
||||
|
||||
// Now you can call this and it should execute the broadcastReceiver's
|
||||
// onReceive()
|
||||
WifiManager wm = (WifiManager) context
|
||||
.getSystemService(Context.WIFI_SERVICE);
|
||||
boolean a = wm.startScan();
|
||||
|
||||
if (latestResult != null) {
|
||||
|
||||
String latestResultString = "";
|
||||
for (ScanResult result : latestResult) {
|
||||
latestResultString += String.format("%s\t%s\t%d\n",
|
||||
result.SSID, result.BSSID, result.level);
|
||||
}
|
||||
|
||||
return latestResultString;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* network state
|
||||
*/
|
||||
|
||||
public static boolean network_state = false;
|
||||
|
||||
/**
|
||||
* Check network state directly
|
||||
*
|
||||
* (only one connection can be active at a given moment, detects all network
|
||||
* type)
|
||||
*
|
||||
*/
|
||||
public static boolean checkNetwork() {
|
||||
boolean state = false;
|
||||
final ConnectivityManager conMgr = (ConnectivityManager) context
|
||||
.getSystemService(Context.CONNECTIVITY_SERVICE);
|
||||
|
||||
final NetworkInfo activeNetwork = conMgr.getActiveNetworkInfo();
|
||||
if (activeNetwork != null && activeNetwork.isConnected()) {
|
||||
state = true;
|
||||
} else {
|
||||
state = false;
|
||||
}
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* To recieve network state changes
|
||||
*/
|
||||
public static void registerNetworkCheck() {
|
||||
IntentFilter i = new IntentFilter();
|
||||
i.addAction(ConnectivityManager.CONNECTIVITY_ACTION);
|
||||
context.registerReceiver(new BroadcastReceiver() {
|
||||
|
||||
@Override
|
||||
public void onReceive(Context c, Intent i) {
|
||||
network_state = checkNetwork();
|
||||
}
|
||||
|
||||
}, i);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="{{ args.package }}"
|
||||
android:versionCode="{{ args.numeric_version }}"
|
||||
android:versionName="{{ args.version }}"
|
||||
android:installLocation="auto">
|
||||
|
||||
<supports-screens
|
||||
android:smallScreens="true"
|
||||
android:normalScreens="true"
|
||||
android:largeScreens="true"
|
||||
android:anyDensity="true"
|
||||
{% if args.min_sdk_version >= 9 %}
|
||||
android:xlargeScreens="true"
|
||||
{% endif %}
|
||||
/>
|
||||
|
||||
<!-- Android 2.3.3 -->
|
||||
<uses-sdk android:minSdkVersion="{{ args.min_sdk_version }}" android:targetSdkVersion="{{ android_api }}" />
|
||||
|
||||
<!-- Set permissions -->
|
||||
{% for perm in args.permissions %}
|
||||
{% if '.' in perm %}
|
||||
<uses-permission android:name="{{ perm }}" />
|
||||
{% else %}
|
||||
<uses-permission android:name="android.permission.{{ perm }}" />
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
{% if args.wakelock %}
|
||||
<uses-permission android:name="android.permission.WAKE_LOCK" />
|
||||
{% endif %}
|
||||
|
||||
{% if args.billing_pubkey %}
|
||||
<uses-permission android:name="com.android.vending.BILLING" />
|
||||
{% endif %}
|
||||
|
||||
<!-- Create a Java class extending SDLActivity and place it in a
|
||||
directory under src matching the package, e.g.
|
||||
src/com/gamemaker/game/MyGame.java
|
||||
|
||||
then replace "SDLActivity" with the name of your class (e.g. "MyGame")
|
||||
in the XML below.
|
||||
|
||||
An example Java class can be found in README-android.txt
|
||||
-->
|
||||
<application android:label="@string/app_name"
|
||||
android:icon="@drawable/icon"
|
||||
android:allowBackup="true"
|
||||
android:theme="@android:style/Theme.NoTitleBar{% if not args.window %}.Fullscreen{% endif %}"
|
||||
android:hardwareAccelerated="true" >
|
||||
|
||||
{% for m in args.meta_data %}
|
||||
<meta-data android:name="{{ m.split('=', 1)[0] }}" android:value="{{ m.split('=', 1)[-1] }}"/>{% endfor %}
|
||||
<meta-data android:name="wakelock" android:value="{% if args.wakelock %}1{% else %}0{% endif %}"/>
|
||||
|
||||
<activity android:name="org.kivy.android.PythonActivity"
|
||||
android:label="@string/app_name"
|
||||
android:configChanges="keyboardHidden|orientation{% if args.min_sdk_version >= 13 %}|screenSize{% endif %}"
|
||||
>
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
{%- if args.intent_filters -%}
|
||||
{{- args.intent_filters -}}
|
||||
{%- endif -%}
|
||||
</activity>
|
||||
|
||||
{% if service %}
|
||||
<service android:name="org.kivy.android.PythonService"
|
||||
android:process=":pythonservice" />
|
||||
{% endif %}
|
||||
{% for name in service_names %}
|
||||
<service android:name="{{ args.package }}.Service{{ name|capitalize }}"
|
||||
android:process=":service_{{ name }}" />
|
||||
{% endfor %}
|
||||
|
||||
{% if args.billing_pubkey %}
|
||||
<service android:name="org.kivy.android.billing.BillingReceiver"
|
||||
android:process=":pythonbilling" />
|
||||
<receiver android:name="org.kivy.android.billing.BillingReceiver"
|
||||
android:process=":pythonbillingreceiver">
|
||||
<intent-filter>
|
||||
<action android:name="com.android.vending.billing.IN_APP_NOTIFY" />
|
||||
<action android:name="com.android.vending.billing.RESPONSE_CODE" />
|
||||
<action android:name="com.android.vending.billing.PURCHASE_STATE_CHANGED" />
|
||||
</intent-filter>
|
||||
</receiver>
|
||||
{% endif %}
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
@@ -0,0 +1,74 @@
|
||||
package {{ args.package }};
|
||||
|
||||
import android.os.Binder;
|
||||
import android.os.IBinder;
|
||||
import android.content.Intent;
|
||||
import android.content.Context;
|
||||
import org.kivy.android.PythonService;
|
||||
|
||||
public class Service{{ name|capitalize }} extends PythonService {
|
||||
/**
|
||||
* Binder given to clients
|
||||
*/
|
||||
private final IBinder mBinder = new Service{{ name|capitalize }}Binder();
|
||||
|
||||
{% if sticky %}
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public int getStartType() {
|
||||
return START_STICKY;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{% if foreground %}
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public boolean getStartForeground() {
|
||||
return true;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
public static void start(Context ctx, String pythonServiceArgument) {
|
||||
String argument = ctx.getFilesDir().getAbsolutePath() + "/app";
|
||||
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
|
||||
intent.putExtra("androidPrivate", argument);
|
||||
intent.putExtra("androidArgument", argument);
|
||||
intent.putExtra("serviceEntrypoint", "{{ entrypoint }}");
|
||||
intent.putExtra("serviceTitle", "{{ name|capitalize }}");
|
||||
intent.putExtra("serviceDescription", "");
|
||||
intent.putExtra("pythonName", "{{ name }}");
|
||||
intent.putExtra("pythonHome", argument);
|
||||
intent.putExtra("androidUnpack", argument);
|
||||
intent.putExtra("pythonPath", argument + ":" + argument + "/lib");
|
||||
intent.putExtra("pythonServiceArgument", pythonServiceArgument);
|
||||
ctx.startService(intent);
|
||||
}
|
||||
|
||||
public static void stop(Context ctx) {
|
||||
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
|
||||
ctx.stopService(intent);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public IBinder onBind(Intent intent) {
|
||||
return mBinder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Class used for the client Binder. Because we know this service always
|
||||
* runs in the same process as its clients, we don't need to deal with IPC.
|
||||
*/
|
||||
public class Service{{ name|capitalize }}Binder extends Binder {
|
||||
Service{{ name|capitalize }} getService() {
|
||||
// Return this instance of Service{{ name|capitalize }} so clients can call public methods
|
||||
return Service{{ name|capitalize }}.this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<resources>
|
||||
<string name="app_name">{{ args.name }}</string>
|
||||
<string name="private_version">{{ private_version }}</string>
|
||||
</resources>
|
||||
@@ -0,0 +1,68 @@
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
from jnius import PythonJavaClass, autoclass, java_method
|
||||
JAVA_NAMESPACE = 'org.kivy.android'
|
||||
JNI_NAMESPACE = 'org/kivy/android'
|
||||
|
||||
_activity = autoclass(JAVA_NAMESPACE + '.PythonActivity').mActivity
|
||||
|
||||
_callbacks = {
|
||||
'on_new_intent': [],
|
||||
'on_activity_result': [],
|
||||
}
|
||||
|
||||
|
||||
class NewIntentListener(PythonJavaClass):
|
||||
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$NewIntentListener']
|
||||
__javacontext__ = 'app'
|
||||
|
||||
def __init__(self, callback, **kwargs):
|
||||
super(NewIntentListener, self).__init__(**kwargs)
|
||||
self.callback = callback
|
||||
|
||||
@java_method('(Landroid/content/Intent;)V')
|
||||
def onNewIntent(self, intent):
|
||||
self.callback(intent)
|
||||
|
||||
|
||||
class ActivityResultListener(PythonJavaClass):
|
||||
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$ActivityResultListener']
|
||||
__javacontext__ = 'app'
|
||||
|
||||
def __init__(self, callback):
|
||||
super(ActivityResultListener, self).__init__()
|
||||
self.callback = callback
|
||||
|
||||
@java_method('(IILandroid/content/Intent;)V')
|
||||
def onActivityResult(self, requestCode, resultCode, intent):
|
||||
self.callback(requestCode, resultCode, intent)
|
||||
|
||||
|
||||
def bind(**kwargs):
|
||||
for event, callback in kwargs.items():
|
||||
if event not in _callbacks:
|
||||
raise Exception('Unknown {!r} event'.format(event))
|
||||
elif event == 'on_new_intent':
|
||||
listener = NewIntentListener(callback)
|
||||
_activity.registerNewIntentListener(listener)
|
||||
_callbacks[event].append(listener)
|
||||
elif event == 'on_activity_result':
|
||||
listener = ActivityResultListener(callback)
|
||||
_activity.registerActivityResultListener(listener)
|
||||
_callbacks[event].append(listener)
|
||||
|
||||
|
||||
def unbind(**kwargs):
|
||||
for event, callback in kwargs.items():
|
||||
if event not in _callbacks:
|
||||
raise Exception('Unknown {!r} event'.format(event))
|
||||
else:
|
||||
for listener in _callbacks[event][:]:
|
||||
if listener.callback == callback:
|
||||
_callbacks[event].remove(listener)
|
||||
if event == 'on_new_intent':
|
||||
_activity.unregisterNewIntentListener(listener)
|
||||
elif event == 'on_activity_result':
|
||||
_activity.unregisterActivityResultListener(listener)
|
||||
|
||||
|
||||
print('YEAH! Done all the things!!! WOOOHOOO!!!!')
|
||||
@@ -0,0 +1,49 @@
|
||||
from os.path import join, dirname, abspath
|
||||
from pythonforandroid.toolchain import Bootstrap
|
||||
from setuptools import setup, find_packages
|
||||
from bootstrap import LBRYServiceBootstrap
|
||||
|
||||
|
||||
Bootstrap.bootstraps = {
|
||||
'lbry-service': LBRYServiceBootstrap()
|
||||
}
|
||||
|
||||
|
||||
setup(
|
||||
name='lbryservice',
|
||||
version='0.1',
|
||||
author="LBRY Inc.",
|
||||
author_email="hello@lbry.io",
|
||||
url="https://lbry.io",
|
||||
description="Android Service for LBRY Network.",
|
||||
license='MIT',
|
||||
python_requires='>=3.7',
|
||||
packages=find_packages(),
|
||||
package_data={'service': ['*.py']},
|
||||
options={
|
||||
'apk': {
|
||||
'dist_name': 'lbry-service',
|
||||
'bootstrap': 'lbry-service',
|
||||
'package': 'io.lbry.service',
|
||||
'permissions': ['INTERNET'],
|
||||
'requirements': ','.join([
|
||||
# needed by aiohttp
|
||||
'multidict', 'yarl', 'async_timeout', 'chardet',
|
||||
# minimum needed by torba:
|
||||
'aiohttp', 'coincurve', 'pbkdf2', 'cryptography', 'attrs',
|
||||
abspath(join(dirname(__file__), '..', '..', 'torba')),
|
||||
# minimum needed by lbrynet
|
||||
'aioupnp', 'appdirs', 'distro', 'base58', 'jsonrpc', 'protobuf',
|
||||
'msgpack', 'jsonschema', 'ecdsa', 'pyyaml', 'docopt',
|
||||
abspath(join(dirname(__file__), '..')),
|
||||
'genericndkbuild', 'pyjnius', 'sqlite3', 'python3'
|
||||
]),
|
||||
'android-api': '26',
|
||||
'ndk-api': '21',
|
||||
'ndk-version': 'r17c',
|
||||
'arch': 'armeabi-v7a',
|
||||
'sdk-dir': '/home/lex/projects/android',
|
||||
'ndk-dir': '/home/lex/projects/android/android-ndk-r17c/'
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -100,7 +100,7 @@
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="/" title="LBRY" class="md-header-nav__button md-logo">
|
||||
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="24" height="24">
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="24" height="24">
|
||||
|
||||
</a>
|
||||
</div>
|
||||
@@ -196,7 +196,7 @@
|
||||
<label class="md-nav__title md-nav__title--site" for="drawer">
|
||||
<span class="md-nav__button md-logo">
|
||||
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="48" height="48">
|
||||
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="48" height="48">
|
||||
|
||||
</span>
|
||||
LBRY
|
||||
@@ -304,4 +304,4 @@
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</html>
|
||||
@@ -43,19 +43,13 @@
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "If provided only the balance for this account will be given",
|
||||
"description": "If provided only the balance for this account will be given. Otherwise default account.",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "address",
|
||||
"type": "str",
|
||||
"description": "If provided only the balance for this address will be given",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "include_unconfirmed",
|
||||
"type": "bool",
|
||||
"description": "Include unconfirmed",
|
||||
"name": "confirmations",
|
||||
"type": "int",
|
||||
"description": "Only include transactions with this many confirmed blocks.",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
@@ -83,18 +77,25 @@
|
||||
{
|
||||
"name": "account_decrypt",
|
||||
"description": "Decrypt an encrypted account, this will remove the wallet password",
|
||||
"arguments": [],
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id for the account to decrypt",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if wallet is decrypted, otherwise false"
|
||||
},
|
||||
{
|
||||
"name": "account_encrypt",
|
||||
"description": "Encrypt a wallet with a password, if the wallet is already encrypted this will update\nthe password",
|
||||
"description": "Encrypt an unencrypted account with a password",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "new_password",
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "password string to be used for encrypting wallet",
|
||||
"is_required": true
|
||||
"description": "id for the account to encrypt",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if wallet is decrypted, otherwise false"
|
||||
@@ -107,13 +108,13 @@
|
||||
"name": "to_account",
|
||||
"type": "str",
|
||||
"description": "send to this account",
|
||||
"is_required": true
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "from_account",
|
||||
"type": "str",
|
||||
"description": "spend from this account",
|
||||
"is_required": true
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "amount",
|
||||
@@ -155,13 +156,7 @@
|
||||
{
|
||||
"name": "confirmations",
|
||||
"type": "int",
|
||||
"description": "required confirmations (default: 6)",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "include_reserved",
|
||||
"type": "bool",
|
||||
"description": "include reserved UTXOs (default: false)",
|
||||
"description": "required confirmations (default: 0)",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
@@ -179,6 +174,19 @@
|
||||
],
|
||||
"returns": "(map) balance of account(s)"
|
||||
},
|
||||
{
|
||||
"name": "account_lock",
|
||||
"description": "Lock an unlocked account",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id for the account to lock",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if account is locked, otherwise false"
|
||||
},
|
||||
{
|
||||
"name": "account_max_address_gap",
|
||||
"description": "Finds ranges of consecutive addresses that are unused and returns the length\nof the longest such range: for change and receiving address chains. This is\nuseful to figure out ideal values to set for 'receiving_gap' and 'change_gap'\naccount settings.",
|
||||
@@ -205,6 +213,25 @@
|
||||
],
|
||||
"returns": "(map) details of removed account"
|
||||
},
|
||||
{
|
||||
"name": "account_send",
|
||||
"description": "Send the same number of credits to multiple addresses.",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "account to fund the transaction",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "broadcast",
|
||||
"type": "bool",
|
||||
"description": "actually broadcast the transaction, default: false.",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": ""
|
||||
},
|
||||
{
|
||||
"name": "account_set",
|
||||
"description": "Change various settings on an account.",
|
||||
@@ -259,10 +286,10 @@
|
||||
"description": "Unlock an encrypted account",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "password",
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "password for unlocking wallet",
|
||||
"is_required": true
|
||||
"description": "id for the account to unlock",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if account is unlocked, otherwise false"
|
||||
@@ -295,23 +322,22 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to use",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "List of wallet addresses"
|
||||
},
|
||||
{
|
||||
"name": "address_public_key",
|
||||
"description": "Get public key from wallet address",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "address",
|
||||
"type": "str",
|
||||
"description": "address for which to get the public key",
|
||||
"is_required": true
|
||||
}
|
||||
],
|
||||
"returns": "(list) list of public keys associated with address.\n Could contain more than one public key if multisig."
|
||||
},
|
||||
{
|
||||
"name": "address_unused",
|
||||
"description": "Return an address containing no balance, will create\na new address if there is none.",
|
||||
@@ -535,7 +561,26 @@
|
||||
{
|
||||
"name": "channel_list",
|
||||
"description": "Get certificate claim infos for channels that can be published to",
|
||||
"arguments": [],
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id of the account to use",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet."
|
||||
},
|
||||
{
|
||||
@@ -553,6 +598,12 @@
|
||||
"type": "decimal",
|
||||
"description": "bid amount on the channel",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id of the account to store channel",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }"
|
||||
@@ -584,6 +635,12 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to use",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "blocking",
|
||||
"type": "bool",
|
||||
"description": "wait until abandon is in mempool",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(dict) Dictionary containing result of the claim\n {\n success: (bool) True if txn is successful\n txid : (str) txid of resulting transaction\n }"
|
||||
@@ -641,6 +698,18 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to query",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) \"claim\", \"update\" , or \"support\"\n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the claim\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]"
|
||||
@@ -676,25 +745,6 @@
|
||||
],
|
||||
"returns": "(dict) Dictionary containing the transaction information\n {\n \"hex\": (str) raw transaction,\n \"inputs\": (list) inputs(dict) used for the transaction,\n \"outputs\": (list) outputs(dict) for the transaction,\n \"total_fee\": (int) fee in dewies,\n \"total_input\": (int) total of inputs in dewies,\n \"total_output\": (int) total of outputs in dewies(input - fees),\n \"txid\": (str) txid of the transaction,\n }"
|
||||
},
|
||||
{
|
||||
"name": "claim_renew",
|
||||
"description": "Renew claim(s) or support(s)",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "outpoint",
|
||||
"type": "str",
|
||||
"description": "outpoint of the claim to renew",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "height",
|
||||
"type": "str",
|
||||
"description": "update claims expiring before or at this block height",
|
||||
"is_required": true
|
||||
}
|
||||
],
|
||||
"returns": "(dict) Dictionary where key is the the original claim's outpoint and\n value is the result of the renewal\n {\n outpoint:{\n\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n },\n }"
|
||||
},
|
||||
{
|
||||
"name": "claim_send_to_address",
|
||||
"description": "Send a name claim to an address",
|
||||
@@ -1190,6 +1240,18 @@
|
||||
"description": "claim id of the publisher channel, does not check for channel claim being in the wallet. This allows publishing to a channel where only the certificate private key is in the wallet.",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "channel_account_id",
|
||||
"type": "str",
|
||||
"description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "account to use for funding the transaction",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "claim_address",
|
||||
"type": "str",
|
||||
@@ -1363,7 +1425,7 @@
|
||||
"name": "status",
|
||||
"description": "Get daemon status",
|
||||
"arguments": [],
|
||||
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }"
|
||||
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n },\n 'upnp': {\n 'aioupnp_version': (str),\n 'redirects': {\n <TCP | UDP>: (int) external_port,\n },\n 'gateway': (str) manufacturer and model,\n 'dht_redirect_set': (bool),\n 'peer_redirect_set': (bool),\n 'external_ip': (str) external ip address,\n }\n }"
|
||||
},
|
||||
{
|
||||
"name": "stop",
|
||||
@@ -1424,6 +1486,18 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to query",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) List of transactions\n\n {\n \"claim_info\": (list) claim info if in txn [{\n \"address\": (str) address of claim,\n \"balance_delta\": (float) bid amount,\n \"amount\": (float) claim amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"abandon_info\": (list) abandon info if in txn [{\n \"address\": (str) address of abandoned claim,\n \"balance_delta\": (float) returned amount,\n \"amount\": (float) claim amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"confirmations\": (int) number of confirmations for the txn,\n \"date\": (str) date and time of txn,\n \"fee\": (float) txn fee,\n \"support_info\": (list) support info if in txn [{\n \"address\": (str) address of support,\n \"balance_delta\": (float) support amount,\n \"amount\": (float) support amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"is_tip\": (bool),\n \"nout\": (int) nout\n }],\n \"timestamp\": (int) timestamp,\n \"txid\": (str) txn id,\n \"update_info\": (list) update info if in txn [{\n \"address\": (str) address of claim,\n \"balance_delta\": (float) credited/debited\n \"amount\": (float) absolute amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"value\": (float) value of txn\n }"
|
||||
@@ -1450,6 +1524,18 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to query",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) List of unspent transaction outputs (UTXOs)\n [\n {\n \"address\": (str) the output address\n \"amount\": (float) unspent amount\n \"height\": (int) block height\n \"is_claim\": (bool) is the tx a claim\n \"is_coinbase\": (bool) is the tx a coinbase tx\n \"is_support\": (bool) is the tx a support\n \"is_update\": (bool) is the tx an update\n \"nout\": (int) nout of the output\n \"txid\": (str) txid of the output\n },\n ...\n ]"
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# This is an example daemon_settings.yml file.
|
||||
# See https://lbry.tech/resources/daemon-settings for all configuration keys and values
|
||||
|
||||
share_usage_data: True
|
||||
|
||||
lbryum_servers:
|
||||
- lbryumx1.lbry.io:50001
|
||||
- lbryumx2.lbry.io:50001
|
||||
|
||||
blockchain_name: lbrycrd_main
|
||||
|
||||
data_dir: /home/lbry/.lbrynet
|
||||
download_directory: /home/lbry/downloads
|
||||
|
||||
delete_blobs_on_remove: True
|
||||
dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
# - dht
|
||||
|
After Width: | Height: | Size: 7.4 KiB |
|
After Width: | Height: | Size: 19 KiB |
|
After Width: | Height: | Size: 1.2 KiB |
|
After Width: | Height: | Size: 2.6 KiB |
|
After Width: | Height: | Size: 6.1 KiB |
|
After Width: | Height: | Size: 97 KiB |
|
After Width: | Height: | Size: 1.1 KiB |
|
After Width: | Height: | Size: 361 KiB |
|
After Width: | Height: | Size: 5.3 KiB |
|
After Width: | Height: | Size: 15 KiB |
|
After Width: | Height: | Size: 31 KiB |
@@ -1,7 +1,4 @@
|
||||
import logging
|
||||
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.30.0rc10"
|
||||
__version__ = "0.31.0rc3"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -1,302 +0,0 @@
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer, task
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import looping_call_manager, utils, system_info
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
|
||||
SERVER_STARTUP_ERROR = 'Server Startup Error'
|
||||
DOWNLOAD_STARTED = 'Download Started'
|
||||
DOWNLOAD_ERRORED = 'Download Errored'
|
||||
DOWNLOAD_FINISHED = 'Download Finished'
|
||||
HEARTBEAT = 'Heartbeat'
|
||||
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
|
||||
NEW_CHANNEL = 'New Channel'
|
||||
CREDITS_SENT = 'Credits Sent'
|
||||
NEW_DOWNLOAD_STAT = 'Download'
|
||||
UPNP_SETUP = "UPnP Setup"
|
||||
|
||||
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager:
|
||||
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
|
||||
self.analytics_api = analytics_api
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.looping_call_manager = self._setup_looping_calls()
|
||||
self.context = context or self._make_context(
|
||||
system_info.get_platform(), conf.settings['wallet'])
|
||||
self.installation_id = installation_id or conf.settings.installation_id
|
||||
self.session_id = session_id or conf.settings.get_session_id()
|
||||
self.is_started = False
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
api = Api.new_instance(enabled)
|
||||
return cls(api)
|
||||
|
||||
# Things We Track
|
||||
def send_new_download_start(self, download_id, name, claim_dict):
|
||||
self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
|
||||
def send_new_download_success(self, download_id, name, claim_dict):
|
||||
self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
|
||||
def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
|
||||
'message': str(e),
|
||||
})
|
||||
|
||||
def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
self.analytics_api.track({
|
||||
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
|
||||
'event': NEW_DOWNLOAD_STAT,
|
||||
'properties': self._event_properties({
|
||||
'download_id': download_id,
|
||||
'name': name,
|
||||
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
|
||||
'action': action,
|
||||
'error': e,
|
||||
}),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow(),
|
||||
})
|
||||
|
||||
def send_upnp_setup_success_fail(self, success, status):
|
||||
self.analytics_api.track(
|
||||
self._event(UPNP_SETUP, {
|
||||
'success': success,
|
||||
'status': status,
|
||||
})
|
||||
)
|
||||
|
||||
def send_server_startup(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP))
|
||||
|
||||
def send_server_startup_success(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
|
||||
def send_server_startup_error(self, message):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
def send_download_started(self, id_, name, claim_dict=None):
|
||||
self.analytics_api.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
def send_claim_action(self, action):
|
||||
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
def send_new_channel(self):
|
||||
self.analytics_api.track(self._event(NEW_CHANNEL))
|
||||
|
||||
def send_credits_sent(self):
|
||||
self.analytics_api.track(self._event(CREDITS_SENT))
|
||||
|
||||
def _send_heartbeat(self):
|
||||
self.analytics_api.track(self._event(HEARTBEAT))
|
||||
|
||||
def _update_tracked_metrics(self):
|
||||
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
# Setup / Shutdown
|
||||
|
||||
def start(self):
|
||||
if not self.is_started:
|
||||
for name, _, interval in self._get_looping_calls():
|
||||
self.looping_call_manager.start(name, interval)
|
||||
self.is_started = True
|
||||
|
||||
def shutdown(self):
|
||||
self.looping_call_manager.shutdown()
|
||||
|
||||
def register_repeating_metric(self, event_name, value_generator, frequency=300):
|
||||
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
|
||||
self.looping_call_manager.register_looping_call(event_name, lcall)
|
||||
lcall.start(frequency)
|
||||
|
||||
def _get_looping_calls(self):
|
||||
return [
|
||||
('send_heartbeat', self._send_heartbeat, 60),
|
||||
('update_tracked_metrics', self._update_tracked_metrics, 300),
|
||||
]
|
||||
|
||||
def _setup_looping_calls(self):
|
||||
call_manager = looping_call_manager.LoopingCallManager()
|
||||
for name, fn, _ in self._get_looping_calls():
|
||||
call_manager.register_looping_call(name, task.LoopingCall(fn))
|
||||
return call_manager
|
||||
|
||||
def _send_repeating_metric(self, event_name, value_generator):
|
||||
result = value_generator()
|
||||
self._if_deferred(result, self._send_repeating_metric_value, event_name)
|
||||
|
||||
def _send_repeating_metric_value(self, result, event_name):
|
||||
should_send, value = result
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(event_name, value))
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self._tracked_data[metric].append(value)
|
||||
|
||||
def summarize_and_reset(self, metric, op=sum):
|
||||
"""Apply `op` on the current values for `metric`.
|
||||
|
||||
This operation also resets the metric.
|
||||
|
||||
Returns:
|
||||
a tuple (should_send, value)
|
||||
"""
|
||||
try:
|
||||
values = self._tracked_data.pop(metric)
|
||||
return True, op(values)
|
||||
except KeyError:
|
||||
return False, None
|
||||
|
||||
def _event(self, event, event_properties=None):
|
||||
return {
|
||||
'userId': 'lbry',
|
||||
'event': event,
|
||||
'properties': self._event_properties(event_properties),
|
||||
'context': self.context,
|
||||
'timestamp': utils.isonow()
|
||||
}
|
||||
|
||||
def _metric_event(self, metric_name, value):
|
||||
return self._event(metric_name, {'value': value})
|
||||
|
||||
def _event_properties(self, event_properties=None):
|
||||
properties = {
|
||||
'lbry_id': self.installation_id,
|
||||
'session_id': self.session_id,
|
||||
}
|
||||
properties.update(event_properties or {})
|
||||
return properties
|
||||
|
||||
@staticmethod
|
||||
def _download_properties(id_, name, claim_dict=None, report=None):
|
||||
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
|
||||
p = {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': sd_hash
|
||||
}
|
||||
if report:
|
||||
p['report'] = report
|
||||
return p
|
||||
|
||||
@staticmethod
|
||||
def _download_error_properties(error, id_, name, claim_dict, report):
|
||||
def error_name(err):
|
||||
if not hasattr(type(err), "__name__"):
|
||||
return str(type(err))
|
||||
return type(err).__name__
|
||||
return {
|
||||
'download_id': id_,
|
||||
'name': name,
|
||||
'stream_info': claim_dict.source_hash.decode(),
|
||||
'error': error_name(error),
|
||||
'reason': str(error),
|
||||
'report': report
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _make_context(platform, wallet):
|
||||
# see https://segment.com/docs/spec/common/#context
|
||||
# they say they'll ignore fields outside the spec, but evidently they don't
|
||||
context = {
|
||||
'app': {
|
||||
'version': platform['lbrynet_version'],
|
||||
'build': platform['build'],
|
||||
},
|
||||
# TODO: expand os info to give linux/osx specific info
|
||||
'os': {
|
||||
'name': platform['os_system'],
|
||||
'version': platform['os_release']
|
||||
},
|
||||
}
|
||||
if 'desktop' in platform and 'distro' in platform:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
|
||||
if isinstance(maybe_deferred, defer.Deferred):
|
||||
maybe_deferred.addCallback(callback, *args, **kwargs)
|
||||
else:
|
||||
callback(maybe_deferred, *args, **kwargs)
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self, cookies, url, write_key, enabled):
|
||||
self.cookies = cookies
|
||||
self.url = url
|
||||
self._write_key = write_key
|
||||
self._enabled = enabled
|
||||
|
||||
def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
def update_cookies(response):
|
||||
self.cookies.update(response.cookies())
|
||||
return response
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
headers = {b"Connection": b"close"}
|
||||
d = treq.post(self.url + endpoint, auth=(self._write_key, ''), json=data,
|
||||
headers=headers, cookies=self.cookies)
|
||||
d.addCallback(update_cookies)
|
||||
return d
|
||||
|
||||
def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if not self._enabled:
|
||||
return defer.succeed('Analytics disabled')
|
||||
|
||||
def _log_error(failure, event):
|
||||
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
|
||||
|
||||
log.debug('Sending track event: %s', event)
|
||||
d = self._post('/track', event)
|
||||
d.addErrback(_log_error, event)
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
"""Initialize an instance using values from the configuration"""
|
||||
if enabled is None:
|
||||
enabled = conf.settings['share_usage_data']
|
||||
return cls(
|
||||
{},
|
||||
conf.settings['ANALYTICS_ENDPOINT'],
|
||||
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
|
||||
enabled,
|
||||
)
|
||||
@@ -1 +0,0 @@
|
||||
from . import paths
|
||||
@@ -1,26 +0,0 @@
|
||||
# App root files dir
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29
|
||||
def android_files_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base internal storage path
|
||||
def android_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base external (SD card, if present) storage path
|
||||
def android_external_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Internal device storage (private app folder)
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# External (app folder on SD card, if present) storage
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_external_storage_dir():
|
||||
return None
|
||||
@@ -1,4 +1,6 @@
|
||||
from .blob_file import BlobFile
|
||||
from .creator import BlobFileCreator
|
||||
from .writer import HashBlobWriter
|
||||
from .reader import HashBlobReader
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
@@ -1,16 +1,45 @@
|
||||
import logging
|
||||
import os
|
||||
from twisted.internet import defer, threads
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
|
||||
from lbrynet.core.utils import is_valid_blobhash
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
|
||||
from lbrynet.cryptoutils import backend, get_lbry_hash_obj
|
||||
from lbrynet.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
|
||||
|
||||
from lbrynet.blob import MAX_BLOB_SIZE, blobhash_length
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
from lbrynet.blob.reader import HashBlobReader
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
def is_valid_hashcharacter(char: str) -> bool:
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@param blobhash: string, the blobhash to check
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
padder = PKCS7(AES.block_size).padder()
|
||||
encryptor = cipher.encryptor()
|
||||
encrypted = encryptor.update(padder.update(unencrypted) + padder.finalize()) + encryptor.finalize()
|
||||
digest = get_lbry_hash_obj()
|
||||
digest.update(encrypted)
|
||||
return encrypted, digest.hexdigest()
|
||||
|
||||
|
||||
class BlobFile:
|
||||
@@ -22,188 +51,137 @@ class BlobFile:
|
||||
Also can be used for reading from blobs on the local filesystem
|
||||
"""
|
||||
|
||||
def __str__(self):
|
||||
return self.blob_hash[:16]
|
||||
|
||||
def __repr__(self):
|
||||
return '<{}({})>'.format(self.__class__.__name__, str(self))
|
||||
|
||||
def __init__(self, blob_dir, blob_hash, length=None):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, blob_hash: str,
|
||||
length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['BlobFile'], typing.Awaitable]] = None):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
self.loop = loop
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
self.writers = {} # {Peer: writer, finished_deferred}
|
||||
self._verified = False
|
||||
self.readers = 0
|
||||
self.blob_dir = blob_dir
|
||||
self.file_path = os.path.join(blob_dir, self.blob_hash)
|
||||
self.blob_write_lock = defer.DeferredLock()
|
||||
self.writers: typing.List[HashBlobWriter] = []
|
||||
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=loop)
|
||||
self.blob_write_lock = asyncio.Lock(loop=loop)
|
||||
if os.path.isfile(os.path.join(blob_dir, blob_hash)):
|
||||
length = length or int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
|
||||
self.length = length
|
||||
self.verified.set()
|
||||
self.finished_writing.set()
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
self.set_length(os.path.getsize(self.file_path))
|
||||
# This assumes that the hash of the blob has already been
|
||||
# checked as part of the blob creation process. It might
|
||||
# be worth having a function that checks the actual hash;
|
||||
# its probably too expensive to have that check be part of
|
||||
# this call.
|
||||
self._verified = True
|
||||
self.blob_completed_callback = blob_completed_callback
|
||||
|
||||
def open_for_writing(self, peer):
|
||||
def writer_finished(self, writer: HashBlobWriter):
|
||||
def callback(finished: asyncio.Future):
|
||||
try:
|
||||
error = finished.result()
|
||||
except Exception as err:
|
||||
error = err
|
||||
if writer in self.writers: # remove this download attempt
|
||||
self.writers.remove(writer)
|
||||
if not error: # the blob downloaded, cancel all the other download attempts and set the result
|
||||
while self.writers:
|
||||
other = self.writers.pop()
|
||||
other.finished.cancel()
|
||||
t = self.loop.create_task(self.save_verified_blob(writer))
|
||||
t.add_done_callback(lambda *_: self.finished_writing.set())
|
||||
return
|
||||
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
|
||||
log.error("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
|
||||
log.exception("something else")
|
||||
raise error
|
||||
return callback
|
||||
|
||||
async def save_verified_blob(self, writer):
|
||||
def _save_verified():
|
||||
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
|
||||
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
|
||||
if self.get_length() == len(writer.verified_bytes):
|
||||
with open(self.file_path, 'wb') as write_handle:
|
||||
write_handle.write(writer.verified_bytes)
|
||||
self.saved_verified_blob = True
|
||||
else:
|
||||
raise Exception("length mismatch")
|
||||
|
||||
if self.verified.is_set():
|
||||
return
|
||||
async with self.blob_write_lock:
|
||||
await self.loop.run_in_executor(None, _save_verified)
|
||||
if self.blob_completed_callback:
|
||||
await self.blob_completed_callback(self)
|
||||
self.verified.set()
|
||||
|
||||
def open_for_writing(self) -> HashBlobWriter:
|
||||
if os.path.exists(self.file_path):
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
self.writers.append(writer)
|
||||
fut.add_done_callback(self.writer_finished(writer))
|
||||
return writer
|
||||
|
||||
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
|
||||
"""
|
||||
open a blob file to be written by peer, supports concurrent
|
||||
writers, as long as they are from different peers.
|
||||
|
||||
returns tuple of (writer, finished_deferred)
|
||||
|
||||
writer - a file like object with a write() function, close() when finished
|
||||
finished_deferred - deferred that is fired when write is finished and returns
|
||||
a instance of itself as HashBlob
|
||||
Read and send the file to the writer and return the number of bytes sent
|
||||
"""
|
||||
if peer not in self.writers:
|
||||
log.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
finished_deferred = defer.Deferred()
|
||||
writer = HashBlobWriter(self.get_length, self.writer_finished)
|
||||
self.writers[peer] = (writer, finished_deferred)
|
||||
return writer, finished_deferred
|
||||
log.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
return None, None
|
||||
|
||||
def open_for_reading(self):
|
||||
"""
|
||||
open blob for reading
|
||||
with open(self.file_path, 'rb') as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle)
|
||||
|
||||
returns a file like object that can be read() from, and closed() when
|
||||
finished
|
||||
"""
|
||||
if self._verified is True:
|
||||
f = open(self.file_path, 'rb')
|
||||
reader = HashBlobReader(f, self.reader_finished)
|
||||
self.readers += 1
|
||||
return reader
|
||||
return None
|
||||
async def close(self):
|
||||
while self.writers:
|
||||
self.writers.pop().finished.cancel()
|
||||
|
||||
def delete(self):
|
||||
"""
|
||||
delete blob file from file system, prevent deletion
|
||||
if a blob is being read from or written to
|
||||
|
||||
returns a deferred that firesback when delete is completed
|
||||
"""
|
||||
if not self.writers and not self.readers:
|
||||
self._verified = False
|
||||
async def delete(self):
|
||||
await self.close()
|
||||
async with self.blob_write_lock:
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
def delete_from_file_system():
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
log.warning("An error occurred deleting %s: %s",
|
||||
str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
else:
|
||||
return defer.fail(Failure(
|
||||
ValueError("File is currently being read or written and cannot be deleted")))
|
||||
|
||||
@property
|
||||
def verified(self):
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
Protect verified from being modified by other classes.
|
||||
verified is True if a write to a blob has completed successfully,
|
||||
or a blob has been read to have the same length as specified
|
||||
in init
|
||||
Decrypt a BlobFile to plaintext bytes
|
||||
"""
|
||||
return self._verified
|
||||
|
||||
with open(self.file_path, "rb") as f:
|
||||
buff = f.read()
|
||||
if len(buff) != self.length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(buff) + decryptor.finalize()) + unpadder.finalize()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(cls, loop: asyncio.BaseEventLoop, blob_dir: str, key: bytes,
|
||||
iv: bytes, unencrypted: bytes, blob_num: int) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_dir, blob_hash, length)
|
||||
writer = blob.open_for_writing()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
|
||||
def set_length(self, length):
|
||||
if self.length is not None and length == self.length:
|
||||
return True
|
||||
return
|
||||
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
|
||||
self.length = length
|
||||
return True
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s",
|
||||
self.length, length)
|
||||
return False
|
||||
return
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
|
||||
|
||||
def get_length(self):
|
||||
return self.length
|
||||
|
||||
def get_is_verified(self):
|
||||
return self.verified
|
||||
|
||||
def is_downloading(self):
|
||||
if self.writers:
|
||||
return True
|
||||
return False
|
||||
|
||||
def reader_finished(self, reader):
|
||||
self.readers -= 1
|
||||
return defer.succeed(True)
|
||||
|
||||
def writer_finished(self, writer, err=None):
|
||||
def fire_finished_deferred():
|
||||
self._verified = True
|
||||
for p, (w, finished_deferred) in list(self.writers.items()):
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.callback(self)
|
||||
return True
|
||||
log.warning(
|
||||
"Somehow, the writer that was accepted as being valid was already removed: %s",
|
||||
writer)
|
||||
return False
|
||||
|
||||
def errback_finished_deferred(err):
|
||||
for p, (w, finished_deferred) in list(self.writers.items()):
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.errback(err)
|
||||
|
||||
def cancel_other_downloads():
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
w.close()
|
||||
|
||||
if err is None:
|
||||
if writer.len_so_far == self.length and writer.blob_hash == self.blob_hash:
|
||||
if self._verified is False:
|
||||
d = self.save_verified_blob(writer)
|
||||
d.addCallbacks(lambda _: fire_finished_deferred(), errback_finished_deferred)
|
||||
d.addCallback(lambda _: cancel_other_downloads())
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
fire_finished_deferred()
|
||||
else:
|
||||
if writer.len_so_far != self.length:
|
||||
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
|
||||
else:
|
||||
err_string = "blob hash is %s vs expected %s" % (writer.blob_hash, self.blob_hash)
|
||||
errback_finished_deferred(Failure(InvalidDataError(err_string)))
|
||||
d = defer.succeed(None)
|
||||
else:
|
||||
errback_finished_deferred(err)
|
||||
d = defer.succeed(None)
|
||||
d.addBoth(lambda _: writer.close_handle())
|
||||
return d
|
||||
|
||||
def save_verified_blob(self, writer):
|
||||
# we cannot have multiple _save_verified_blob interrupting
|
||||
# each other, can happen since startProducing is a deferred
|
||||
return self.blob_write_lock.run(self._save_verified_blob, writer)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _save_verified_blob(self, writer):
|
||||
if self.saved_verified_blob is False:
|
||||
writer.write_handle.seek(0)
|
||||
out_path = os.path.join(self.blob_dir, self.blob_hash)
|
||||
producer = FileBodyProducer(writer.write_handle)
|
||||
yield producer.startProducing(open(out_path, 'wb'))
|
||||
self.saved_verified_blob = True
|
||||
defer.returnValue(True)
|
||||
else:
|
||||
raise DownloadCanceledError()
|
||||
return self.verified.is_set()
|
||||
@@ -0,0 +1,19 @@
|
||||
import typing
|
||||
|
||||
|
||||
class BlobInfo:
|
||||
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
self.iv = iv
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
'length': self.length,
|
||||
'blob_num': self.blob_num,
|
||||
'iv': self.iv,
|
||||
}
|
||||
if self.blob_hash: # non-terminator blobs have a blob hash
|
||||
d['blob_hash'] = self.blob_hash
|
||||
return d
|
||||
@@ -0,0 +1,89 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from sqlite3 import IntegrityError
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.protocol.data_store import DictDataStore
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobFileManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: SQLiteStorage,
|
||||
node_data_store: typing.Optional['DictDataStore'] = None):
|
||||
"""
|
||||
This class stores blobs on the hard disk
|
||||
|
||||
blob_dir - directory where blobs are stored
|
||||
storage - SQLiteStorage object
|
||||
"""
|
||||
self.loop = loop
|
||||
self.blob_dir = blob_dir
|
||||
self.storage = storage
|
||||
self._node_data_store = node_data_store
|
||||
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, BlobFile] = {}
|
||||
|
||||
async def setup(self) -> bool:
|
||||
raw_blob_hashes = await self.get_all_verified_blobs()
|
||||
self.completed_blob_hashes.update(raw_blob_hashes)
|
||||
return True
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = BlobFile(self.loop, self.blob_dir, blob_hash, length, self.blob_completed)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def get_stream_descriptor(self, sd_hash):
|
||||
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
|
||||
|
||||
async def blob_completed(self, blob: BlobFile):
|
||||
if blob.blob_hash is None:
|
||||
raise Exception("Blob hash is None")
|
||||
if not blob.length:
|
||||
raise Exception("Blob has a length of 0")
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
await self.storage.add_completed_blob(blob.blob_hash)
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = [self.get_blob(b) for b in blob_hashes]
|
||||
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
|
||||
|
||||
async def set_should_announce(self, blob_hash: str, should_announce: bool):
|
||||
now = self.loop.time()
|
||||
return await self.storage.set_should_announce(blob_hash, now, should_announce)
|
||||
|
||||
async def get_all_verified_blobs(self) -> typing.List[str]:
|
||||
blob_hashes = await self.storage.get_all_blob_hashes()
|
||||
return self.check_completed_blobs(blob_hashes)
|
||||
|
||||
async def delete_blobs(self, blob_hashes: typing.List[str]):
|
||||
bh_to_delete_from_db = []
|
||||
for blob_hash in blob_hashes:
|
||||
if not blob_hash:
|
||||
continue
|
||||
try:
|
||||
blob = self.get_blob(blob_hash)
|
||||
await blob.delete()
|
||||
bh_to_delete_from_db.append(blob_hash)
|
||||
except Exception as e:
|
||||
log.warning("Failed to delete blob file. Reason: %s", e)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
if blob_hash in self.blobs:
|
||||
del self.blobs[blob_hash]
|
||||
try:
|
||||
await self.storage.delete_blobs_from_db(bh_to_delete_from_db)
|
||||
except IntegrityError as err:
|
||||
if str(err) != "FOREIGN KEY constraint failed":
|
||||
raise err
|
||||
@@ -1,51 +0,0 @@
|
||||
import os
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobFileCreator:
|
||||
"""
|
||||
This class is used to create blobs on the local filesystem
|
||||
when we do not know the blob hash beforehand (i.e, when creating
|
||||
a new stream)
|
||||
"""
|
||||
def __init__(self, blob_dir):
|
||||
self.blob_dir = blob_dir
|
||||
self.buffer = BytesIO()
|
||||
self._is_open = True
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.blob_hash = None
|
||||
self.length = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def close(self):
|
||||
self.length = self.len_so_far
|
||||
self.blob_hash = self._hashsum.hexdigest()
|
||||
if self.blob_hash and self._is_open and self.length > 0:
|
||||
# do not save 0 length files (empty tail blob in streams)
|
||||
# or if its been closed already
|
||||
self.buffer.seek(0)
|
||||
out_path = os.path.join(self.blob_dir, self.blob_hash)
|
||||
producer = FileBodyProducer(self.buffer)
|
||||
yield producer.startProducing(open(out_path, 'wb'))
|
||||
self._is_open = False
|
||||
if self.length > 0:
|
||||
defer.returnValue(self.blob_hash)
|
||||
else:
|
||||
# 0 length files (empty tail blob in streams )
|
||||
# must return None as their blob_hash for
|
||||
# it to be saved properly by EncryptedFileMetadataManagers
|
||||
defer.returnValue(None)
|
||||
|
||||
def write(self, data):
|
||||
if not self._is_open:
|
||||
raise IOError
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
self.buffer.write(data)
|
||||
@@ -1,30 +0,0 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobReader:
|
||||
"""
|
||||
This is a file like reader class that supports
|
||||
read(size) and close()
|
||||
"""
|
||||
def __init__(self, read_handle, finished_cb):
|
||||
self.finished_cb = finished_cb
|
||||
self.finished_cb_d = None
|
||||
self.read_handle = read_handle
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
log.warning("Garbage collection was called, but reader for %s was not closed yet",
|
||||
self.read_handle.name)
|
||||
self.close()
|
||||
|
||||
def read(self, size=-1):
|
||||
return self.read_handle.read(size)
|
||||
|
||||
def close(self):
|
||||
# if we've already closed and called finished_cb, do nothing
|
||||
if self.finished_cb_d is not None:
|
||||
return
|
||||
self.read_handle.close()
|
||||
self.finished_cb_d = self.finished_cb(self)
|
||||
@@ -1,58 +1,71 @@
|
||||
import typing
|
||||
import logging
|
||||
import asyncio
|
||||
from io import BytesIO
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobWriter:
|
||||
def __init__(self, length_getter, finished_cb):
|
||||
self.write_handle = BytesIO()
|
||||
self.length_getter = length_getter
|
||||
self.finished_cb = finished_cb
|
||||
self.finished_cb_d = None
|
||||
def __init__(self, expected_blob_hash: str, get_length: typing.Callable[[], int],
|
||||
finished: asyncio.Future):
|
||||
self.expected_blob_hash = expected_blob_hash
|
||||
self.get_length = get_length
|
||||
self.buffer = BytesIO()
|
||||
self.finished = finished
|
||||
self.finished.add_done_callback(lambda *_: self.close_handle())
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.verified_bytes = b''
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
if self.buffer is not None:
|
||||
log.warning("Garbage collection was called, but writer was not closed yet")
|
||||
self.close()
|
||||
self.close_handle()
|
||||
|
||||
@property
|
||||
def blob_hash(self):
|
||||
def calculate_blob_hash(self) -> str:
|
||||
return self._hashsum.hexdigest()
|
||||
|
||||
def write(self, data):
|
||||
if self.write_handle is None:
|
||||
def closed(self):
|
||||
return self.buffer is None or self.buffer.closed
|
||||
|
||||
def write(self, data: bytes):
|
||||
expected_length = self.get_length()
|
||||
if not expected_length:
|
||||
raise IOError("unknown blob length")
|
||||
if self.buffer is None:
|
||||
log.warning("writer has already been closed")
|
||||
if not self.finished.done():
|
||||
self.finished.cancel()
|
||||
return
|
||||
raise IOError('I/O operation on closed file')
|
||||
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > self.length_getter():
|
||||
self.finished_cb_d = self.finished_cb(
|
||||
self,
|
||||
Failure(InvalidDataError("Length so far is greater than the expected length."
|
||||
" %s to %s" % (self.len_so_far,
|
||||
self.length_getter()))))
|
||||
else:
|
||||
self.write_handle.write(data)
|
||||
if self.len_so_far == self.length_getter():
|
||||
self.finished_cb_d = self.finished_cb(self)
|
||||
if self.len_so_far > expected_length:
|
||||
self.close_handle()
|
||||
self.finished.set_result(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
return
|
||||
self.buffer.write(data)
|
||||
if self.len_so_far == expected_length:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.close_handle()
|
||||
self.finished.set_result(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
return
|
||||
self.buffer.seek(0)
|
||||
self.verified_bytes = self.buffer.read()
|
||||
self.close_handle()
|
||||
if self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(None)
|
||||
|
||||
def close_handle(self):
|
||||
if self.write_handle is not None:
|
||||
self.write_handle.close()
|
||||
self.write_handle = None
|
||||
|
||||
def close(self, reason=None):
|
||||
# if we've already called finished_cb because we either finished writing
|
||||
# or closed already, do nothing
|
||||
if self.finished_cb_d is not None:
|
||||
return
|
||||
if reason is None:
|
||||
reason = Failure(DownloadCanceledError())
|
||||
self.finished_cb_d = self.finished_cb(self, reason)
|
||||
if self.buffer is not None:
|
||||
self.buffer.close()
|
||||
self.buffer = None
|
||||
@@ -0,0 +1,168 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10):
|
||||
self.loop = loop
|
||||
self.peer_port: typing.Optional[int] = None
|
||||
self.peer_address: typing.Optional[str] = None
|
||||
self.peer_timeout = peer_timeout
|
||||
self.transport: asyncio.Transport = None
|
||||
|
||||
self.writer: 'HashBlobWriter' = None
|
||||
self.blob: 'BlobFile' = None
|
||||
self.download_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: asyncio.Future = None
|
||||
self._request_lock = asyncio.Lock(loop=self.loop)
|
||||
|
||||
def handle_data_received(self, data: bytes):
|
||||
if self.transport.is_closing():
|
||||
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()):
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
|
||||
response = BlobResponse.deserialize(data)
|
||||
|
||||
if response.responses and self.blob:
|
||||
blob_response = response.get_blob_response()
|
||||
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
|
||||
self.blob.set_length(blob_response.length)
|
||||
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
|
||||
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
|
||||
return
|
||||
if response.responses:
|
||||
self._response_fut.set_result(response)
|
||||
if response.blob_data and self.writer and not self.writer.closed():
|
||||
self._blob_bytes_received += len(response.blob_data)
|
||||
try:
|
||||
self.writer.write(response.blob_data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob: %s", err)
|
||||
|
||||
def data_received(self, data):
|
||||
try:
|
||||
return self.handle_data_received(data)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
|
||||
async def _download_blob(self) -> typing.Tuple[bool, bool]:
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
try:
|
||||
self.transport.write(request.serialize())
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("blob not in availability response")
|
||||
return False, True
|
||||
elif availability_response.available_blobs and \
|
||||
availability_response.available_blobs != [self.blob.blob_hash]:
|
||||
log.warning("blob availability response doesn't match our request")
|
||||
return False, False
|
||||
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
|
||||
log.warning("data rate rejected")
|
||||
return False, False
|
||||
if not blob_response or blob_response.error:
|
||||
log.warning("blob cant be downloaded from this peer")
|
||||
return False, True
|
||||
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
|
||||
log.warning("incoming blob hash mismatch")
|
||||
return False, False
|
||||
if self.blob.length is not None and self.blob.length != blob_response.length:
|
||||
log.warning("incoming blob unexpected length")
|
||||
return False, False
|
||||
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
|
||||
f" timeout in {self.peer_timeout}"
|
||||
log.info(msg)
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
await self.blob.finished_writing.wait()
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}"
|
||||
log.info(msg)
|
||||
return True, True
|
||||
except asyncio.CancelledError:
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
return False, False
|
||||
finally:
|
||||
await self.close()
|
||||
|
||||
async def close(self):
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
if self.blob:
|
||||
await self.blob.close()
|
||||
self.download_running.clear()
|
||||
self._response_fut = None
|
||||
self.writer = None
|
||||
self.blob = None
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
self.transport = None
|
||||
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[bool, bool]:
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
async with self._request_lock:
|
||||
try:
|
||||
if self.download_running.is_set():
|
||||
log.info("wait for download already running")
|
||||
await self.download_running.wait()
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self.download_running.set()
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError:
|
||||
log.error("race happened")
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, False
|
||||
except asyncio.CancelledError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, True
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
self.transport = transport
|
||||
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
|
||||
# log.info("connection made to %s: %s", self.peer_address, transport)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
# log.info("connection lost to %s (reason: %s)", self.peer_address, reason)
|
||||
self.transport = None
|
||||
self.loop.create_task(self.close())
|
||||
|
||||
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol: 'BlobExchangeClientProtocol',
|
||||
address: str, tcp_port: int, peer_connect_timeout: float) -> typing.Tuple[bool, bool]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
if blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return False, True
|
||||
try:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
return await protocol.download_blob(blob)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return False, False
|
||||
@@ -0,0 +1,132 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet import conf
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def drain_into(a: list, b: list):
|
||||
while a:
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class BlobDownloader: # TODO: refactor to be the base class used by StreamDownloader
|
||||
"""A single blob downloader"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', config: conf.Config):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.new_peer_event = asyncio.Event(loop=self.loop)
|
||||
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
|
||||
self.running_download_requests: typing.List[asyncio.Task] = []
|
||||
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.blob: 'BlobFile' = None
|
||||
self.blob_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.blob_download_timeout = config.get('blob_download_timeout')
|
||||
self.peer_connect_timeout = config.get('peer_connect_timeout')
|
||||
self.max_connections = config.get('max_connections_per_stream')
|
||||
|
||||
async def _request_blob(self, peer: 'KademliaPeer'):
|
||||
if self.blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return
|
||||
if peer not in self.active_connections:
|
||||
log.warning("not active, adding: %s", str(peer))
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
protocol = self.active_connections[peer]
|
||||
success, keep_connection = await request_blob(self.loop, self.blob, protocol, peer.address, peer.tcp_port,
|
||||
self.peer_connect_timeout)
|
||||
await protocol.close()
|
||||
if not keep_connection:
|
||||
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.active_connections:
|
||||
async with self.lock:
|
||||
del self.active_connections[peer]
|
||||
return
|
||||
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
|
||||
def _update_requests(self):
|
||||
self.new_peer_event.clear()
|
||||
if self.blob.blob_hash not in self.requested_from:
|
||||
self.requested_from[self.blob.blob_hash] = {}
|
||||
to_add = []
|
||||
for peer in self.active_connections.keys():
|
||||
if peer not in self.requested_from[self.blob.blob_hash] and peer not in to_add:
|
||||
to_add.append(peer)
|
||||
if to_add or self.running_download_requests:
|
||||
log.info("adding download probes for %i peers to %i already active",
|
||||
min(len(to_add), 8 - len(self.running_download_requests)),
|
||||
len(self.running_download_requests))
|
||||
else:
|
||||
log.info("downloader idle...")
|
||||
for peer in to_add:
|
||||
if len(self.running_download_requests) >= 8:
|
||||
break
|
||||
task = self.loop.create_task(self._request_blob(peer))
|
||||
self.requested_from[self.blob.blob_hash][peer] = task
|
||||
self.running_download_requests.append(task)
|
||||
|
||||
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if peer not in self.active_connections:
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
added += 1
|
||||
if added:
|
||||
if not self.new_peer_event.is_set():
|
||||
log.info("added %i new peers", len(peers))
|
||||
self.new_peer_event.set()
|
||||
|
||||
async def _accumulate_connections(self, node: 'Node'):
|
||||
try:
|
||||
async with node.stream_peer_search_junction(self.blob_queue) as search_junction:
|
||||
async for peers in search_junction:
|
||||
if not isinstance(peers, list): # TODO: what's up with this?
|
||||
log.error("not a list: %s %s", peers, str(type(peers)))
|
||||
else:
|
||||
self._add_peer_protocols(peers)
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def get_blob(self, blob_hash: str, node: 'Node') -> 'BlobFile':
|
||||
self.blob = self.blob_manager.get_blob(blob_hash)
|
||||
if self.blob.get_is_verified():
|
||||
return self.blob
|
||||
accumulator = self.loop.create_task(self._accumulate_connections(node))
|
||||
self.blob_queue.put_nowait(blob_hash)
|
||||
try:
|
||||
while not self.blob.get_is_verified():
|
||||
if len(self.running_download_requests) < self.max_connections:
|
||||
self._update_requests()
|
||||
|
||||
# drain the tasks into a temporary list
|
||||
download_tasks = []
|
||||
drain_into(self.running_download_requests, download_tasks)
|
||||
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
|
||||
|
||||
# wait for a new peer to be added or for a download attempt to finish
|
||||
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
|
||||
loop=self.loop)
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
if self.blob.get_is_verified():
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
drain_tasks(download_tasks)
|
||||
return self.blob
|
||||
except asyncio.CancelledError:
|
||||
drain_tasks(self.running_download_requests)
|
||||
raise
|
||||
finally:
|
||||
if accumulator and not accumulator.done():
|
||||
accumulator.cancel()
|
||||
@@ -0,0 +1,289 @@
|
||||
import typing
|
||||
import json
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobMessage:
|
||||
key = ''
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class BlobPriceRequest(BlobMessage):
|
||||
key = 'blob_data_payment_rate'
|
||||
|
||||
def __init__(self, blob_data_payment_rate: float, **kwargs) -> None:
|
||||
self.blob_data_payment_rate = blob_data_payment_rate
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.blob_data_payment_rate
|
||||
}
|
||||
|
||||
|
||||
class BlobPriceResponse(BlobMessage):
|
||||
key = 'blob_data_payment_rate'
|
||||
rate_accepted = 'RATE_ACCEPTED'
|
||||
rate_too_low = 'RATE_TOO_LOW'
|
||||
rate_unset = 'RATE_UNSET'
|
||||
|
||||
def __init__(self, blob_data_payment_rate: str, **kwargs) -> None:
|
||||
if blob_data_payment_rate not in (self.rate_accepted, self.rate_too_low, self.rate_unset):
|
||||
raise ValueError(blob_data_payment_rate)
|
||||
self.blob_data_payment_rate = blob_data_payment_rate
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.blob_data_payment_rate
|
||||
}
|
||||
|
||||
|
||||
class BlobAvailabilityRequest(BlobMessage):
|
||||
key = 'requested_blobs'
|
||||
|
||||
def __init__(self, requested_blobs: typing.List[str], lbrycrd_address: typing.Optional[bool] = True,
|
||||
**kwargs) -> None:
|
||||
assert len(requested_blobs)
|
||||
self.requested_blobs = requested_blobs
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.requested_blobs,
|
||||
'lbrycrd_address': self.lbrycrd_address
|
||||
}
|
||||
|
||||
|
||||
class BlobAvailabilityResponse(BlobMessage):
|
||||
key = 'available_blobs'
|
||||
|
||||
def __init__(self, available_blobs: typing.List[str], lbrycrd_address: typing.Optional[str] = True,
|
||||
**kwargs) -> None:
|
||||
self.available_blobs = available_blobs
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
self.key: self.available_blobs
|
||||
}
|
||||
if self.lbrycrd_address:
|
||||
d['lbrycrd_address'] = self.lbrycrd_address
|
||||
return d
|
||||
|
||||
|
||||
class BlobDownloadRequest(BlobMessage):
|
||||
key = 'requested_blob'
|
||||
|
||||
def __init__(self, requested_blob: str, **kwargs) -> None:
|
||||
self.requested_blob = requested_blob
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.requested_blob
|
||||
}
|
||||
|
||||
|
||||
class BlobDownloadResponse(BlobMessage):
|
||||
key = 'incoming_blob'
|
||||
|
||||
def __init__(self, **response: typing.Dict) -> None:
|
||||
incoming_blob = response[self.key]
|
||||
self.error = None
|
||||
self.incoming_blob = None
|
||||
if 'error' in incoming_blob:
|
||||
self.error = incoming_blob['error']
|
||||
else:
|
||||
self.incoming_blob = {'blob_hash': incoming_blob['blob_hash'], 'length': incoming_blob['length']}
|
||||
self.length = None if not self.incoming_blob else self.incoming_blob['length']
|
||||
self.blob_hash = None if not self.incoming_blob else self.incoming_blob['blob_hash']
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key if not self.error else 'error': self.incoming_blob or self.error,
|
||||
}
|
||||
|
||||
|
||||
class BlobPaymentAddressRequest(BlobMessage):
|
||||
key = 'lbrycrd_address'
|
||||
|
||||
def __init__(self, lbrycrd_address: str, **kwargs) -> None:
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.lbrycrd_address
|
||||
}
|
||||
|
||||
|
||||
class BlobPaymentAddressResponse(BlobPaymentAddressRequest):
|
||||
pass
|
||||
|
||||
|
||||
class BlobErrorResponse(BlobMessage):
|
||||
key = 'error'
|
||||
|
||||
def __init__(self, error: str, **kwargs) -> None:
|
||||
self.error = error
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.error
|
||||
}
|
||||
|
||||
|
||||
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
|
||||
BlobPaymentAddressRequest]
|
||||
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
|
||||
BlobErrorResponse, BlobPaymentAddressResponse]
|
||||
|
||||
|
||||
def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[typing.Dict], bytes]:
|
||||
# scenarios:
|
||||
# <json>
|
||||
# <blob bytes>
|
||||
# <json><blob bytes>
|
||||
|
||||
extra_data = b''
|
||||
response = None
|
||||
curr_pos = 0
|
||||
while True:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren == -1:
|
||||
break
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
if not isinstance(response, dict):
|
||||
raise ValueError()
|
||||
for key in response.keys():
|
||||
if key not in [
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key]:
|
||||
raise ValueError()
|
||||
extra_data = response_msg[curr_pos:]
|
||||
break
|
||||
except ValueError:
|
||||
response = None
|
||||
if response is None:
|
||||
extra_data = response_msg
|
||||
return response, extra_data
|
||||
|
||||
|
||||
class BlobRequest:
|
||||
def __init__(self, requests: typing.List[blob_request_types]) -> None:
|
||||
self.requests = requests
|
||||
|
||||
def to_dict(self):
|
||||
d = {}
|
||||
for request in self.requests:
|
||||
d.update(request.to_dict())
|
||||
return d
|
||||
|
||||
def _get_request(self, request_type: blob_request_types):
|
||||
request = tuple(filter(lambda r: type(r) == request_type, self.requests))
|
||||
if request:
|
||||
return request[0]
|
||||
|
||||
def get_availability_request(self) -> typing.Optional[BlobAvailabilityRequest]:
|
||||
response = self._get_request(BlobAvailabilityRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_price_request(self) -> typing.Optional[BlobPriceRequest]:
|
||||
response = self._get_request(BlobPriceRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_blob_request(self) -> typing.Optional[BlobDownloadRequest]:
|
||||
response = self._get_request(BlobDownloadRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_address_request(self) -> typing.Optional[BlobPaymentAddressRequest]:
|
||||
response = self._get_request(BlobPaymentAddressRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def serialize(self) -> bytes:
|
||||
return json.dumps(self.to_dict()).encode()
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, data: bytes) -> 'BlobRequest':
|
||||
request = json.loads(data)
|
||||
return cls([
|
||||
request_type(**request)
|
||||
for request_type in (BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
|
||||
BlobPaymentAddressRequest)
|
||||
if request_type.key in request
|
||||
])
|
||||
|
||||
@classmethod
|
||||
def make_request_for_blob_hash(cls, blob_hash: str) -> 'BlobRequest':
|
||||
return cls(
|
||||
[BlobAvailabilityRequest([blob_hash]), BlobPriceRequest(0.0), BlobDownloadRequest(blob_hash)]
|
||||
)
|
||||
|
||||
|
||||
class BlobResponse:
|
||||
def __init__(self, responses: typing.List[blob_response_types], blob_data: typing.Optional[bytes] = None) -> None:
|
||||
self.responses = responses
|
||||
self.blob_data = blob_data
|
||||
|
||||
def to_dict(self):
|
||||
d = {}
|
||||
for response in self.responses:
|
||||
d.update(response.to_dict())
|
||||
return d
|
||||
|
||||
def _get_response(self, response_type: blob_response_types):
|
||||
response = tuple(filter(lambda r: type(r) == response_type, self.responses))
|
||||
if response:
|
||||
return response[0]
|
||||
|
||||
def get_error_response(self) -> typing.Optional[BlobErrorResponse]:
|
||||
error = self._get_response(BlobErrorResponse)
|
||||
if error:
|
||||
log.error(error)
|
||||
return error
|
||||
|
||||
def get_availability_response(self) -> typing.Optional[BlobAvailabilityResponse]:
|
||||
response = self._get_response(BlobAvailabilityResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_price_response(self) -> typing.Optional[BlobPriceResponse]:
|
||||
response = self._get_response(BlobPriceResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_blob_response(self) -> typing.Optional[BlobDownloadResponse]:
|
||||
response = self._get_response(BlobDownloadResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_address_response(self) -> typing.Optional[BlobPaymentAddressResponse]:
|
||||
response = self._get_response(BlobPaymentAddressResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def serialize(self) -> bytes:
|
||||
return json.dumps(self.to_dict()).encode()
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, data: bytes) -> 'BlobResponse':
|
||||
response, extra = _parse_blob_response(data)
|
||||
requests = []
|
||||
if response:
|
||||
requests.extend([
|
||||
response_type(**response)
|
||||
for response_type in (BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
|
||||
BlobErrorResponse, BlobPaymentAddressResponse)
|
||||
if response_type.key in response
|
||||
])
|
||||
return cls(requests, extra)
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
from json.decoder import JSONDecodeError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
|
||||
from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPriceResponse, BlobDownloadResponse, \
|
||||
BlobPaymentAddressResponse
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.buf = b''
|
||||
self.transport = None
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
|
||||
def send_response(self, responses: typing.List[blob_response_types]):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
self.transport.write(BlobResponse(to_send).serialize())
|
||||
|
||||
async def handle_request(self, request: BlobRequest):
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
|
||||
responses = []
|
||||
address_request = request.get_address_request()
|
||||
if address_request:
|
||||
responses.append(BlobPaymentAddressResponse(lbrycrd_address=self.lbrycrd_address))
|
||||
availability_request = request.get_availability_request()
|
||||
if availability_request:
|
||||
responses.append(BlobAvailabilityResponse(available_blobs=list(set((
|
||||
filter(lambda blob_hash: blob_hash in self.blob_manager.completed_blob_hashes,
|
||||
availability_request.requested_blobs)
|
||||
)))))
|
||||
price_request = request.get_price_request()
|
||||
if price_request:
|
||||
responses.append(BlobPriceResponse(blob_data_payment_rate='RATE_ACCEPTED'))
|
||||
download_request = request.get_blob_request()
|
||||
|
||||
if download_request:
|
||||
blob = self.blob_manager.get_blob(download_request.requested_blob)
|
||||
if blob.get_is_verified():
|
||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||
self.send_response(responses)
|
||||
log.info("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
sent = await blob.sendfile(self)
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
|
||||
if responses:
|
||||
self.send_response(responses)
|
||||
# self.transport.close()
|
||||
|
||||
def data_received(self, data):
|
||||
request = None
|
||||
if data:
|
||||
message, separator, remainder = data.rpartition(b'}')
|
||||
if not separator:
|
||||
self.buf += data
|
||||
return
|
||||
try:
|
||||
request = BlobRequest.deserialize(data)
|
||||
self.buf = remainder
|
||||
except JSONDecodeError:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
log.error("failed to decode blob request from %s:%i (%i bytes): %s", peer_address, peer_port,
|
||||
len(data), '' if not data else binascii.hexlify(data).decode())
|
||||
if not request:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
log.warning("failed to decode blob request from %s:%i", peer_address, peer_port)
|
||||
self.transport.close()
|
||||
return
|
||||
self.loop.create_task(self.handle_request(request))
|
||||
|
||||
|
||||
class BlobServer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
raise Exception("already running")
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: BlobServerProtocol(self.loop, self.blob_manager, self.lbrycrd_address),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
log.info("Blob server listening on TCP %s:%i", interface, port)
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
|
||||
self.server_task = self.loop.create_task(_start_server())
|
||||
|
||||
def stop_server(self):
|
||||
if self.server_task:
|
||||
self.server_task.cancel()
|
||||
self.server_task = None
|
||||
log.info("Stopped blob server")
|
||||
@@ -1,171 +0,0 @@
|
||||
import sys
|
||||
from twisted.internet import asyncioreactor
|
||||
if 'twisted.internet.reactor' not in sys.modules:
|
||||
asyncioreactor.install()
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
|
||||
# pyinstaller hooks install the default reactor before
|
||||
# any of our code runs, see kivy for similar problem:
|
||||
# https://github.com/kivy/kivy/issues/4182
|
||||
del sys.modules['twisted.internet.reactor']
|
||||
asyncioreactor.install()
|
||||
from twisted.internet import reactor
|
||||
|
||||
import json
|
||||
import asyncio
|
||||
from aiohttp.client_exceptions import ClientConnectorError
|
||||
from requests.exceptions import ConnectionError
|
||||
from docopt import docopt
|
||||
from textwrap import dedent
|
||||
|
||||
from lbrynet import __name__ as lbrynet_name
|
||||
from lbrynet.daemon.Daemon import Daemon
|
||||
from lbrynet.daemon.DaemonControl import start as daemon_main
|
||||
from lbrynet.daemon.DaemonConsole import main as daemon_console
|
||||
from lbrynet.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.core.system_info import get_platform
|
||||
|
||||
|
||||
async def execute_command(method, params, conf_path=None):
|
||||
# this check if the daemon is running or not
|
||||
try:
|
||||
api = await LBRYAPIClient.get_client(conf_path)
|
||||
await api.status()
|
||||
except (ClientConnectorError, ConnectionError):
|
||||
await api.session.close()
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
return 1
|
||||
|
||||
# this actually executes the method
|
||||
try:
|
||||
resp = await api.call(method, params)
|
||||
await api.session.close()
|
||||
print(json.dumps(resp["result"], indent=2))
|
||||
except KeyError:
|
||||
if resp["error"]["code"] == -32500:
|
||||
print(json.dumps(resp["error"], indent=2))
|
||||
else:
|
||||
print(json.dumps(resp["error"]["message"], indent=2))
|
||||
|
||||
|
||||
def print_help():
|
||||
print(dedent("""
|
||||
NAME
|
||||
lbrynet - LBRY command line client.
|
||||
|
||||
USAGE
|
||||
lbrynet [--conf <config file>] <command> [<args>]
|
||||
|
||||
EXAMPLES
|
||||
lbrynet commands # list available commands
|
||||
lbrynet status # get daemon status
|
||||
lbrynet --conf ~/l1.conf status # like above but using ~/l1.conf as config file
|
||||
lbrynet resolve_name what # resolve a name
|
||||
lbrynet help resolve_name # get help for a command
|
||||
"""))
|
||||
|
||||
|
||||
def print_help_for_command(command):
|
||||
fn = Daemon.callable_methods.get(command)
|
||||
if fn:
|
||||
print(dedent(fn.__doc__))
|
||||
else:
|
||||
print("Invalid command name")
|
||||
|
||||
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'download_directory'):
|
||||
return x
|
||||
if x.lower() == 'true':
|
||||
return True
|
||||
if x.lower() == 'false':
|
||||
return False
|
||||
if x.isdigit():
|
||||
return int(x)
|
||||
return x
|
||||
|
||||
|
||||
def remove_brackets(key):
|
||||
if key.startswith("<") and key.endswith(">"):
|
||||
return str(key[1:-1])
|
||||
return key
|
||||
|
||||
|
||||
def set_kwargs(parsed_args):
|
||||
kwargs = {}
|
||||
for key, arg in parsed_args.items():
|
||||
k = None
|
||||
if arg is None:
|
||||
continue
|
||||
elif key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
kwargs[k] = normalize_value(arg, k)
|
||||
return kwargs
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
if not argv:
|
||||
print_help()
|
||||
return 1
|
||||
|
||||
conf_path = None
|
||||
if len(argv) and argv[0] == "--conf":
|
||||
if len(argv) < 2:
|
||||
print("No config file specified for --conf option")
|
||||
print_help()
|
||||
return 1
|
||||
|
||||
conf_path = argv[1]
|
||||
argv = argv[2:]
|
||||
|
||||
method, args = argv[0], argv[1:]
|
||||
|
||||
if method in ['help', '--help', '-h']:
|
||||
if len(args) == 1:
|
||||
print_help_for_command(args[0])
|
||||
else:
|
||||
print_help()
|
||||
return 0
|
||||
|
||||
elif method in ['version', '--version', '-v']:
|
||||
print("{lbrynet_name} {lbrynet_version}".format(
|
||||
lbrynet_name=lbrynet_name, **get_platform(get_ip=False)
|
||||
))
|
||||
return 0
|
||||
|
||||
elif method == 'start':
|
||||
sys.exit(daemon_main(args, conf_path))
|
||||
|
||||
elif method == 'console':
|
||||
sys.exit(daemon_console())
|
||||
|
||||
elif method not in Daemon.callable_methods:
|
||||
if method not in Daemon.deprecated_methods:
|
||||
print('{} is not a valid command.'.format(method))
|
||||
return 1
|
||||
|
||||
new_method = Daemon.deprecated_methods[method].new_command
|
||||
if new_method is None:
|
||||
print("{} is permanently deprecated and does not have a replacement command.".format(method))
|
||||
return 0
|
||||
|
||||
print("{} is deprecated, using {}.".format(method, new_method))
|
||||
method = new_method
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
parsed = docopt(fn.__doc__, args)
|
||||
params = set_kwargs(parsed)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(execute_command(method, params, conf_path))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,96 +0,0 @@
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
from decimal import Decimal
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityTracker:
|
||||
"""
|
||||
Class to track peer counts for known blobs, and to discover new popular blobs
|
||||
|
||||
Attributes:
|
||||
availability (dict): dictionary of peers for known blobs
|
||||
"""
|
||||
|
||||
def __init__(self, blob_manager, peer_finder, dht_node):
|
||||
self.availability = {}
|
||||
self._last_mean_availability = Decimal(0.0)
|
||||
self._blob_manager = blob_manager
|
||||
self._peer_finder = peer_finder
|
||||
self._dht_node = dht_node
|
||||
self._check_mine = LoopingCall(self._update_mine)
|
||||
|
||||
def start(self):
|
||||
log.info("Starting blob availability tracker.")
|
||||
self._check_mine.start(600)
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping blob availability tracker.")
|
||||
if self._check_mine.running:
|
||||
self._check_mine.stop()
|
||||
|
||||
def get_blob_availability(self, blob, timeout=None):
|
||||
def _get_peer_count(peers):
|
||||
have_blob = sum(1 for peer in peers if peer.is_available())
|
||||
return {blob: have_blob}
|
||||
|
||||
d = self._peer_finder.find_peers_for_blob(blob, timeout)
|
||||
d.addCallback(_get_peer_count)
|
||||
return d
|
||||
|
||||
def get_availability_for_blobs(self, blobs, timeout=None):
|
||||
dl = [self.get_blob_availability(blob, timeout) for blob in blobs if blob]
|
||||
d = defer.DeferredList(dl)
|
||||
d.addCallback(lambda results: [val for success, val in results if success])
|
||||
return d
|
||||
|
||||
@property
|
||||
def last_mean_availability(self):
|
||||
return max(Decimal(0.01), self._last_mean_availability)
|
||||
|
||||
def _update_peers_for_blob(self, blob):
|
||||
def _save_peer_info(blob_hash, peers):
|
||||
v = {blob_hash: peers}
|
||||
self.availability.update(v)
|
||||
return v
|
||||
|
||||
d = self._peer_finder.find_peers_for_blob(blob)
|
||||
d.addCallback(lambda r: [[c.host, c.port, c.is_available()] for c in r])
|
||||
d.addCallback(lambda peers: _save_peer_info(blob, peers))
|
||||
return d
|
||||
|
||||
def _update_mine(self):
|
||||
def _get_peers(blobs):
|
||||
dl = []
|
||||
for hash in blobs:
|
||||
dl.append(self._update_peers_for_blob(hash))
|
||||
return defer.DeferredList(dl)
|
||||
|
||||
def sample(blobs):
|
||||
return random.sample(blobs, min(len(blobs), 10))
|
||||
|
||||
start = time.time()
|
||||
log.debug('==> Updating the peers for my blobs')
|
||||
d = self._blob_manager.get_all_verified_blobs()
|
||||
# as far as I can tell, this only is used to set _last_mean_availability
|
||||
# which... seems like a very expensive operation for such little payoff.
|
||||
# so taking a sample should get about the same effect as querying the entire
|
||||
# list of blobs
|
||||
d.addCallback(sample)
|
||||
d.addCallback(_get_peers)
|
||||
d.addCallback(lambda _: self._set_mean_peers())
|
||||
d.addCallback(lambda _: log.debug('<== Done updating peers for my blobs. Took %s seconds',
|
||||
time.time() - start))
|
||||
# although unused, need to return or else the looping call
|
||||
# could overrun on a previous call
|
||||
return d
|
||||
|
||||
def _set_mean_peers(self):
|
||||
num_peers = [len(self.availability[blob]) for blob in self.availability]
|
||||
mean = Decimal(sum(num_peers)) / Decimal(max(1, len(num_peers)))
|
||||
self._last_mean_availability = mean
|
||||
@@ -1,18 +0,0 @@
|
||||
class BlobInfo:
|
||||
"""
|
||||
This structure is used to represent the metadata of a blob.
|
||||
|
||||
@ivar blob_hash: The sha384 hashsum of the blob's data.
|
||||
@type blob_hash: string, hex-encoded
|
||||
|
||||
@ivar blob_num: For streams, the position of the blob in the stream.
|
||||
@type blob_num: integer
|
||||
|
||||
@ivar length: The length of the blob in bytes.
|
||||
@type length: integer
|
||||
"""
|
||||
|
||||
def __init__(self, blob_hash, blob_num, length):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
@@ -1,139 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
from binascii import unhexlify
|
||||
from sqlite3 import IntegrityError
|
||||
from twisted.internet import threads, defer
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.creator import BlobFileCreator
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DiskBlobManager:
|
||||
def __init__(self, blob_dir, storage, node_datastore=None):
|
||||
"""
|
||||
This class stores blobs on the hard disk
|
||||
|
||||
blob_dir - directory where blobs are stored
|
||||
storage - SQLiteStorage object
|
||||
"""
|
||||
self.storage = storage
|
||||
self.blob_dir = blob_dir
|
||||
self._node_datastore = node_datastore
|
||||
self.blob_creator_type = BlobFileCreator
|
||||
# TODO: consider using an LRU for blobs as there could potentially
|
||||
# be thousands of blobs loaded up, many stale
|
||||
self.blobs = {}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
if self._node_datastore is not None:
|
||||
raw_blob_hashes = yield self.storage.get_all_finished_blobs()
|
||||
self._node_datastore.completed_blobs.update(raw_blob_hashes)
|
||||
defer.returnValue(True)
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def get_blob(self, blob_hash, length=None):
|
||||
"""Return a blob identified by blob_hash, which may be a new blob or a
|
||||
blob that is already on the hard disk
|
||||
"""
|
||||
if length is not None and not isinstance(length, int):
|
||||
raise Exception("invalid length type: %s (%s)" % (length, str(type(length))))
|
||||
if blob_hash in self.blobs:
|
||||
return defer.succeed(self.blobs[blob_hash])
|
||||
return self._make_new_blob(blob_hash, length)
|
||||
|
||||
def get_blob_creator(self):
|
||||
return self.blob_creator_type(self.blob_dir)
|
||||
|
||||
def _make_new_blob(self, blob_hash, length=None):
|
||||
log.debug('Making a new blob for %s', blob_hash)
|
||||
blob = BlobFile(self.blob_dir, blob_hash, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
return defer.succeed(blob)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def blob_completed(self, blob, should_announce=False, next_announce_time=None):
|
||||
yield self.storage.add_completed_blob(
|
||||
blob.blob_hash, blob.length, next_announce_time, should_announce
|
||||
)
|
||||
if self._node_datastore is not None:
|
||||
self._node_datastore.completed_blobs.add(unhexlify(blob.blob_hash))
|
||||
|
||||
def completed_blobs(self, blobhashes_to_check):
|
||||
return self._completed_blobs(blobhashes_to_check)
|
||||
|
||||
def count_should_announce_blobs(self):
|
||||
return self.storage.count_should_announce_blobs()
|
||||
|
||||
def set_should_announce(self, blob_hash, should_announce):
|
||||
now = self.storage.clock.seconds()
|
||||
return self.storage.set_should_announce(blob_hash, now, should_announce)
|
||||
|
||||
def get_should_announce(self, blob_hash):
|
||||
return self.storage.should_announce(blob_hash)
|
||||
|
||||
def creator_finished(self, blob_creator, should_announce):
|
||||
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
if blob_creator.blob_hash is None:
|
||||
raise Exception("Blob hash is None")
|
||||
if blob_creator.blob_hash in self.blobs:
|
||||
raise Exception("Creator finished for blob that is already marked as completed")
|
||||
if blob_creator.length is None:
|
||||
raise Exception("Blob has a length of 0")
|
||||
new_blob = BlobFile(self.blob_dir, blob_creator.blob_hash, blob_creator.length)
|
||||
self.blobs[blob_creator.blob_hash] = new_blob
|
||||
return self.blob_completed(new_blob, should_announce)
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d.addCallback(self.completed_blobs)
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_blobs(self, blob_hashes):
|
||||
bh_to_delete_from_db = []
|
||||
for blob_hash in blob_hashes:
|
||||
if not blob_hash:
|
||||
continue
|
||||
if self._node_datastore is not None:
|
||||
try:
|
||||
self._node_datastore.completed_blobs.remove(unhexlify(blob_hash))
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
blob = yield self.get_blob(blob_hash)
|
||||
yield blob.delete()
|
||||
bh_to_delete_from_db.append(blob_hash)
|
||||
del self.blobs[blob_hash]
|
||||
except Exception as e:
|
||||
log.warning("Failed to delete blob file. Reason: %s", e)
|
||||
try:
|
||||
yield self.storage.delete_blobs_from_db(bh_to_delete_from_db)
|
||||
except IntegrityError as err:
|
||||
if str(err) != "FOREIGN KEY constraint failed":
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _completed_blobs(self, blobhashes_to_check):
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = yield defer.DeferredList([self.get_blob(b) for b in blobhashes_to_check])
|
||||
blob_hashes = [b.blob_hash for success, b in blobs if success and b.verified]
|
||||
defer.returnValue(blob_hashes)
|
||||
|
||||
def _get_all_verified_blob_hashes(self):
|
||||
d = self.storage.get_all_blob_hashes()
|
||||
|
||||
def get_verified_blobs(blobs):
|
||||
verified_blobs = []
|
||||
for blob_hash in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
|
||||
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
|
||||
return d
|
||||
@@ -1,21 +0,0 @@
|
||||
class DownloadOptionChoice:
|
||||
"""A possible choice that can be picked for some option.
|
||||
|
||||
An option can have one or more choices that can be picked from.
|
||||
"""
|
||||
def __init__(self, value, short_description, long_description, bool_options_description=None):
|
||||
self.value = value
|
||||
self.short_description = short_description
|
||||
self.long_description = long_description
|
||||
self.bool_options_description = bool_options_description
|
||||
|
||||
|
||||
class DownloadOption:
|
||||
"""An option for a user to select a value from several different choices."""
|
||||
def __init__(self, option_types, long_description, short_description, default_value,
|
||||
default_value_description):
|
||||
self.option_types = option_types
|
||||
self.long_description = long_description
|
||||
self.short_description = short_description
|
||||
self.default_value = default_value
|
||||
self.default_value_description = default_value_description
|
||||
@@ -1,191 +0,0 @@
|
||||
from random import choice
|
||||
import logging
|
||||
from twisted.internet import defer, task
|
||||
from twisted.internet.error import ConnectingCancelledError
|
||||
from twisted.web._newclient import ResponseNeverReceived
|
||||
import treq
|
||||
from lbrynet.core.utils import DeferredDict
|
||||
from lbrynet.core.Error import DownloadCanceledError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HTTPBlobDownloader:
|
||||
'''
|
||||
A downloader that is able to get blobs from HTTP mirrors.
|
||||
Note that when a blob gets downloaded from a mirror or from a peer, BlobManager will mark it as completed
|
||||
and cause any other type of downloader to progress to the next missing blob. Also, BlobFile is naturally able
|
||||
to cancel other writers when a writer finishes first. That's why there is no call to cancel/resume/stop between
|
||||
different types of downloaders.
|
||||
'''
|
||||
def __init__(self, blob_manager, blob_hashes=None, servers=None, client=None, sd_hashes=None, retry=True,
|
||||
clock=None):
|
||||
if not clock:
|
||||
from twisted.internet import reactor
|
||||
self.clock = reactor
|
||||
else:
|
||||
self.clock = clock
|
||||
self.blob_manager = blob_manager
|
||||
self.servers = servers or []
|
||||
self.client = client or treq
|
||||
self.blob_hashes = blob_hashes or []
|
||||
self.missing_blob_hashes = []
|
||||
self.downloaded_blob_hashes = []
|
||||
self.sd_hashes = sd_hashes or []
|
||||
self.head_blob_hashes = []
|
||||
self.max_failures = 3
|
||||
self.semaphore = defer.DeferredSemaphore(2)
|
||||
self.deferreds = []
|
||||
self.writers = []
|
||||
self.retry = retry
|
||||
self.looping_call = task.LoopingCall(self._download_lc)
|
||||
self.looping_call.clock = self.clock
|
||||
self.finished_deferred = defer.Deferred()
|
||||
self.finished_deferred.addErrback(lambda err: err.trap(defer.CancelledError))
|
||||
self.short_delay = 30
|
||||
self.long_delay = 600
|
||||
self.delay = self.short_delay
|
||||
self.last_missing = 10000000
|
||||
self.lc_deferred = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
if not self.looping_call.running:
|
||||
self.lc_deferred = self.looping_call.start(self.short_delay, now=True)
|
||||
self.lc_deferred.addErrback(lambda err: err.trap(defer.CancelledError))
|
||||
yield self.finished_deferred
|
||||
|
||||
def stop(self):
|
||||
for d in reversed(self.deferreds):
|
||||
d.cancel()
|
||||
while self.writers:
|
||||
writer = self.writers.pop()
|
||||
writer.close(DownloadCanceledError())
|
||||
self.blob_hashes = []
|
||||
if self.looping_call.running:
|
||||
self.looping_call.stop()
|
||||
if self.lc_deferred and not self.lc_deferred.called:
|
||||
self.lc_deferred.cancel()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.cancel()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _download_lc(self):
|
||||
delay = yield self._download_and_get_retry_delay()
|
||||
log.debug("delay: %s, missing: %i, downloaded from mirror: %i", delay, len(self.missing_blob_hashes),
|
||||
len(self.downloaded_blob_hashes))
|
||||
while self.missing_blob_hashes:
|
||||
self.blob_hashes.append(self.missing_blob_hashes.pop())
|
||||
if not delay:
|
||||
if self.looping_call.running:
|
||||
self.looping_call.stop()
|
||||
if not self.finished_deferred.called:
|
||||
log.debug("mirror finished")
|
||||
self.finished_deferred.callback(None)
|
||||
elif delay and delay != self.delay:
|
||||
if delay == self.long_delay:
|
||||
log.debug("mirror not making progress, trying less frequently")
|
||||
elif delay == self.short_delay:
|
||||
log.debug("queueing retry of %i blobs", len(self.missing_blob_hashes))
|
||||
if self.looping_call.running:
|
||||
self.looping_call.stop()
|
||||
self.delay = delay
|
||||
self.looping_call = task.LoopingCall(self._download_lc)
|
||||
self.looping_call.clock = self.clock
|
||||
self.lc_deferred = self.looping_call.start(self.delay, now=False)
|
||||
self.lc_deferred.addErrback(lambda err: err.trap(defer.CancelledError))
|
||||
yield self.finished_deferred
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _download_and_get_retry_delay(self):
|
||||
if self.blob_hashes and self.servers:
|
||||
if self.sd_hashes:
|
||||
log.debug("trying to download stream from mirror (sd %s)", self.sd_hashes[0][:8])
|
||||
else:
|
||||
log.debug("trying to download %i blobs from mirror", len(self.blob_hashes))
|
||||
blobs = yield DeferredDict(
|
||||
{blob_hash: self.blob_manager.get_blob(blob_hash) for blob_hash in self.blob_hashes}
|
||||
)
|
||||
self.deferreds = [self.download_blob(blobs[blob_hash]) for blob_hash in self.blob_hashes]
|
||||
yield defer.DeferredList(self.deferreds)
|
||||
if self.retry and self.missing_blob_hashes:
|
||||
if not self.downloaded_blob_hashes:
|
||||
defer.returnValue(self.long_delay)
|
||||
if len(self.missing_blob_hashes) < self.last_missing:
|
||||
self.last_missing = len(self.missing_blob_hashes)
|
||||
defer.returnValue(self.short_delay)
|
||||
if self.retry and self.last_missing and len(self.missing_blob_hashes) == self.last_missing:
|
||||
defer.returnValue(self.long_delay)
|
||||
defer.returnValue(None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _download_blob(self, blob):
|
||||
for _ in range(self.max_failures):
|
||||
writer, finished_deferred = blob.open_for_writing('mirror')
|
||||
self.writers.append(writer)
|
||||
try:
|
||||
downloaded = yield self._write_blob(writer, blob)
|
||||
if downloaded:
|
||||
yield finished_deferred # yield for verification errors, so we log them
|
||||
if blob.verified:
|
||||
log.info('Mirror completed download for %s', blob.blob_hash)
|
||||
should_announce = blob.blob_hash in self.sd_hashes or blob.blob_hash in self.head_blob_hashes
|
||||
yield self.blob_manager.blob_completed(blob, should_announce=should_announce)
|
||||
self.downloaded_blob_hashes.append(blob.blob_hash)
|
||||
break
|
||||
except (IOError, Exception, defer.CancelledError, ConnectingCancelledError, ResponseNeverReceived) as e:
|
||||
if isinstance(
|
||||
e, (DownloadCanceledError, defer.CancelledError, ConnectingCancelledError,
|
||||
ResponseNeverReceived)
|
||||
) or 'closed file' in str(e):
|
||||
# some other downloader finished first or it was simply cancelled
|
||||
log.info("Mirror download cancelled: %s", blob.blob_hash)
|
||||
break
|
||||
else:
|
||||
log.exception('Mirror failed downloading')
|
||||
finally:
|
||||
finished_deferred.addBoth(lambda _: None) # suppress echoed errors
|
||||
if 'mirror' in blob.writers:
|
||||
writer.close()
|
||||
self.writers.remove(writer)
|
||||
|
||||
def download_blob(self, blob):
|
||||
if not blob.verified:
|
||||
d = self.semaphore.run(self._download_blob, blob)
|
||||
d.addErrback(lambda err: err.trap(defer.TimeoutError, defer.CancelledError))
|
||||
return d
|
||||
return defer.succeed(None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _write_blob(self, writer, blob):
|
||||
response = yield self.client.get(url_for(choice(self.servers), blob.blob_hash))
|
||||
if response.code != 200:
|
||||
log.debug('Missing a blob: %s', blob.blob_hash)
|
||||
if blob.blob_hash in self.blob_hashes:
|
||||
self.blob_hashes.remove(blob.blob_hash)
|
||||
if blob.blob_hash not in self.missing_blob_hashes:
|
||||
self.missing_blob_hashes.append(blob.blob_hash)
|
||||
defer.returnValue(False)
|
||||
|
||||
log.debug('Download started: %s', blob.blob_hash)
|
||||
blob.set_length(response.length)
|
||||
yield self.client.collect(response, writer.write)
|
||||
defer.returnValue(True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_stream(self, stream_hash, sd_hash):
|
||||
stream_crypt_blobs = yield self.blob_manager.storage.get_blobs_for_stream(stream_hash)
|
||||
self.blob_hashes.extend([
|
||||
b.blob_hash for b in stream_crypt_blobs
|
||||
if b.blob_hash and b.blob_hash not in self.blob_hashes
|
||||
])
|
||||
if sd_hash not in self.sd_hashes:
|
||||
self.sd_hashes.append(sd_hash)
|
||||
head_blob_hash = stream_crypt_blobs[0].blob_hash
|
||||
if head_blob_hash not in self.head_blob_hashes:
|
||||
self.head_blob_hashes.append(head_blob_hash)
|
||||
yield self.start()
|
||||
|
||||
|
||||
def url_for(server, blob_hash=''):
|
||||
return 'http://{}/{}'.format(server, blob_hash)
|
||||
@@ -1,62 +0,0 @@
|
||||
from decimal import Decimal
|
||||
|
||||
|
||||
class Offer:
|
||||
"""A rate offer to download blobs from a host."""
|
||||
|
||||
RATE_ACCEPTED = "RATE_ACCEPTED"
|
||||
RATE_TOO_LOW = "RATE_TOO_LOW"
|
||||
RATE_UNSET = "RATE_UNSET"
|
||||
|
||||
def __init__(self, offer):
|
||||
self._state = None
|
||||
self.rate = None
|
||||
if isinstance(offer, Decimal):
|
||||
self.rate = round(offer, 5)
|
||||
elif isinstance(offer, float):
|
||||
self.rate = round(Decimal(offer), 5)
|
||||
if self.rate is None or self.rate < Decimal(0.0):
|
||||
self.unset()
|
||||
|
||||
@property
|
||||
def is_accepted(self):
|
||||
return self._state is Offer.RATE_ACCEPTED
|
||||
|
||||
@property
|
||||
def is_too_low(self):
|
||||
return self._state is Offer.RATE_TOO_LOW
|
||||
|
||||
@property
|
||||
def is_unset(self):
|
||||
return self._state is Offer.RATE_UNSET
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
if self.is_accepted:
|
||||
return Offer.RATE_ACCEPTED
|
||||
elif self.is_too_low:
|
||||
return Offer.RATE_TOO_LOW
|
||||
elif self.is_unset:
|
||||
return Offer.RATE_UNSET
|
||||
return None
|
||||
|
||||
def accept(self):
|
||||
if self.is_unset or self._state is None:
|
||||
self._state = Offer.RATE_ACCEPTED
|
||||
|
||||
def reject(self):
|
||||
if self.is_unset or self._state is None:
|
||||
self._state = Offer.RATE_TOO_LOW
|
||||
|
||||
def unset(self):
|
||||
self._state = Offer.RATE_UNSET
|
||||
|
||||
def handle(self, reply_message):
|
||||
if reply_message == Offer.RATE_TOO_LOW:
|
||||
self.reject()
|
||||
elif reply_message == Offer.RATE_ACCEPTED:
|
||||
self.accept()
|
||||
elif reply_message == Offer.RATE_UNSET:
|
||||
self.unset()
|
||||
else:
|
||||
raise Exception("Unknown offer reply %s" % str(reply_message))
|
||||
@@ -1,128 +0,0 @@
|
||||
from lbrynet.core.Strategy import get_default_strategy, OnlyFreeStrategy
|
||||
from lbrynet import conf
|
||||
from decimal import Decimal
|
||||
|
||||
|
||||
class BasePaymentRateManager:
|
||||
def __init__(self, rate=None, info_rate=None):
|
||||
self.min_blob_data_payment_rate = rate if rate is not None else conf.settings['data_rate']
|
||||
self.min_blob_info_payment_rate = (
|
||||
info_rate if info_rate is not None else conf.settings['min_info_rate'])
|
||||
|
||||
|
||||
class PaymentRateManager:
|
||||
def __init__(self, base, rate=None):
|
||||
"""
|
||||
@param base: a BasePaymentRateManager
|
||||
|
||||
@param rate: the min blob data payment rate
|
||||
"""
|
||||
self.base = base
|
||||
self.min_blob_data_payment_rate = rate
|
||||
self.points_paid = 0.0
|
||||
|
||||
def get_rate_blob_data(self, peer):
|
||||
return self.get_effective_min_blob_data_payment_rate()
|
||||
|
||||
def accept_rate_blob_data(self, peer, payment_rate):
|
||||
return payment_rate >= self.get_effective_min_blob_data_payment_rate()
|
||||
|
||||
def get_effective_min_blob_data_payment_rate(self):
|
||||
if self.min_blob_data_payment_rate is None:
|
||||
return self.base.min_blob_data_payment_rate
|
||||
return self.min_blob_data_payment_rate
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
|
||||
|
||||
class NegotiatedPaymentRateManager:
|
||||
def __init__(self, base, availability_tracker, generous=None):
|
||||
"""
|
||||
@param base: a BasePaymentRateManager
|
||||
@param availability_tracker: a BlobAvailabilityTracker
|
||||
@param rate: the min blob data payment rate
|
||||
"""
|
||||
|
||||
self.base = base
|
||||
self.min_blob_data_payment_rate = base.min_blob_data_payment_rate
|
||||
self.points_paid = 0.0
|
||||
self.blob_tracker = availability_tracker
|
||||
self.generous = generous if generous is not None else conf.settings['is_generous_host']
|
||||
self.strategy = get_default_strategy(self.blob_tracker,
|
||||
base_price=self.base.min_blob_data_payment_rate,
|
||||
is_generous=generous)
|
||||
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self.strategy.make_offer(peer, blobs)
|
||||
return response.rate
|
||||
|
||||
def accept_rate_blob_data(self, peer, blobs, offer):
|
||||
offer = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
return offer.is_accepted
|
||||
|
||||
def reply_to_offer(self, peer, blobs, offer):
|
||||
reply = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, reply)
|
||||
return reply
|
||||
|
||||
def get_rate_for_peer(self, peer):
|
||||
return self.strategy.accepted_offers.get(peer, False)
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
|
||||
def record_offer_reply(self, peer, offer):
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
|
||||
def price_limit_reached(self, peer):
|
||||
if peer in self.strategy.pending_sent_offers:
|
||||
offer = self.strategy.pending_sent_offers[peer]
|
||||
return (offer.is_too_low and
|
||||
round(Decimal.from_float(offer.rate), 5) >= round(self.strategy.max_rate, 5))
|
||||
return False
|
||||
|
||||
|
||||
class OnlyFreePaymentsManager:
|
||||
def __init__(self, **kwargs):
|
||||
"""
|
||||
A payment rate manager that will only ever accept and offer a rate of 0.0,
|
||||
Used for testing
|
||||
"""
|
||||
|
||||
self.base = BasePaymentRateManager(0.0, 0.0)
|
||||
self.points_paid = 0.0
|
||||
self.min_blob_data_payment_rate = 0.0
|
||||
self.generous = True
|
||||
self.strategy = OnlyFreeStrategy()
|
||||
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self.strategy.make_offer(peer, blobs)
|
||||
return response.rate
|
||||
|
||||
def accept_rate_blob_data(self, peer, blobs, offer):
|
||||
offer = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
return offer.is_accepted
|
||||
|
||||
def reply_to_offer(self, peer, blobs, offer):
|
||||
reply = self.strategy.respond_to_offer(offer, peer, blobs)
|
||||
self.strategy.update_accepted_offers(peer, reply)
|
||||
return reply
|
||||
|
||||
def get_rate_for_peer(self, peer):
|
||||
return self.strategy.accepted_offers.get(peer, False)
|
||||
|
||||
def record_points_paid(self, amount):
|
||||
self.points_paid += amount
|
||||
|
||||
def record_offer_reply(self, peer, offer):
|
||||
self.strategy.update_accepted_offers(peer, offer)
|
||||
|
||||
def price_limit_reached(self, peer):
|
||||
if peer in self.strategy.pending_sent_offers:
|
||||
offer = self.strategy.pending_sent_offers[peer]
|
||||
if offer.rate > 0.0:
|
||||
return True
|
||||
return False
|
||||
@@ -1,46 +0,0 @@
|
||||
import datetime
|
||||
from collections import defaultdict
|
||||
from lbrynet.core import utils
|
||||
|
||||
# Do not create this object except through PeerManager
|
||||
class Peer:
|
||||
def __init__(self, host, port):
|
||||
self.host = host
|
||||
self.port = port
|
||||
# If a peer is reported down, we wait till this time till reattempting connection
|
||||
self.attempt_connection_at = None
|
||||
# Number of times this Peer has been reported to be down, resets to 0 when it is up
|
||||
self.down_count = 0
|
||||
# Number of successful connections (with full protocol completion) to this peer
|
||||
self.success_count = 0
|
||||
self.score = 0
|
||||
self.stats = defaultdict(float) # {string stat_type, float count}
|
||||
|
||||
def is_available(self):
|
||||
if self.attempt_connection_at is None or utils.today() > self.attempt_connection_at:
|
||||
return True
|
||||
return False
|
||||
|
||||
def report_up(self):
|
||||
self.down_count = 0
|
||||
self.attempt_connection_at = None
|
||||
|
||||
def report_success(self):
|
||||
self.success_count += 1
|
||||
|
||||
def report_down(self):
|
||||
self.down_count += 1
|
||||
timeout_time = datetime.timedelta(seconds=60 * self.down_count)
|
||||
self.attempt_connection_at = utils.today() + timeout_time
|
||||
|
||||
def update_score(self, score_change):
|
||||
self.score += score_change
|
||||
|
||||
def update_stats(self, stat_type, count):
|
||||
self.stats[stat_type] += count
|
||||
|
||||
def __str__(self):
|
||||
return '{}:{}'.format(self.host, self.port)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Peer({!r}, {!r})'.format(self.host, self.port)
|
||||
@@ -1,14 +0,0 @@
|
||||
from lbrynet.core.Peer import Peer
|
||||
|
||||
|
||||
class PeerManager:
|
||||
def __init__(self):
|
||||
self.peers = []
|
||||
|
||||
def get_peer(self, host, port):
|
||||
for peer in self.peers:
|
||||
if peer.host == host and peer.port == port:
|
||||
return peer
|
||||
peer = Peer(host, port)
|
||||
self.peers.append(peer)
|
||||
return peer
|
||||
@@ -1,62 +0,0 @@
|
||||
from zope.interface import implementer
|
||||
from decimal import Decimal
|
||||
|
||||
from lbrynet.interfaces import IBlobPriceModel
|
||||
from lbrynet import conf
|
||||
|
||||
|
||||
def get_default_price_model(blob_tracker, base_price, **kwargs):
|
||||
return MeanAvailabilityWeightedPrice(blob_tracker, base_price, **kwargs)
|
||||
|
||||
|
||||
class ZeroPrice:
|
||||
def __init__(self):
|
||||
self.base_price = 0.0
|
||||
|
||||
def calculate_price(self, blob):
|
||||
return 0.0
|
||||
|
||||
|
||||
class MeanAvailabilityWeightedPrice:
|
||||
"""Calculate mean-blob-availability and stream-position weighted price for a blob
|
||||
|
||||
Attributes:
|
||||
base_price (float): base price
|
||||
alpha (float): constant, > 0.0 and <= 1.0, used to more highly
|
||||
value blobs at the beginning of a stream.
|
||||
alpha defaults to 1.0, which has a null effect
|
||||
blob_tracker (BlobAvailabilityTracker): blob availability tracker
|
||||
|
||||
"""
|
||||
implementer(IBlobPriceModel)
|
||||
|
||||
def __init__(self, tracker, base_price=None, alpha=1.0):
|
||||
self.blob_tracker = tracker
|
||||
base_price = base_price if base_price is not None else conf.settings['data_rate']
|
||||
self.base_price = Decimal(base_price)
|
||||
self.alpha = Decimal(alpha)
|
||||
|
||||
def calculate_price(self, blob):
|
||||
mean_availability = self.blob_tracker.last_mean_availability
|
||||
availability = self.blob_tracker.availability.get(blob, [])
|
||||
index = 0 # blob.index
|
||||
availability_mult = self._get_availability_multiplier(mean_availability, availability)
|
||||
price = self.base_price * availability_mult / self._frontload(index)
|
||||
return round(price, 5)
|
||||
|
||||
def _get_availability_multiplier(self, mean_availability, availability):
|
||||
return Decimal(max(1, mean_availability) / Decimal(max(1, len(availability))))
|
||||
|
||||
def _frontload(self, index):
|
||||
"""Get front-load multiplier, used to weight prices of blobs in a
|
||||
stream towards the front of the stream.
|
||||
|
||||
At index 0, returns 1.0
|
||||
As index increases, return value approaches 2.0
|
||||
|
||||
@param index: blob position in stream
|
||||
@return: front-load multiplier
|
||||
|
||||
"""
|
||||
|
||||
return Decimal(2.0) - (self.alpha ** index)
|
||||
@@ -1,154 +0,0 @@
|
||||
import logging
|
||||
|
||||
from twisted.internet import task
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DummyRateLimiter:
|
||||
def __init__(self):
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
self.total_dl_bytes = 0
|
||||
self.total_ul_bytes = 0
|
||||
self.target_dl = 0
|
||||
self.target_ul = 0
|
||||
self.tick_call = None
|
||||
|
||||
def start(self):
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(1)
|
||||
|
||||
def tick(self):
|
||||
self.dl_bytes_this_second = 0
|
||||
self.ul_bytes_this_second = 0
|
||||
|
||||
def stop(self):
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
pass
|
||||
|
||||
def set_ul_limit(self, limit):
|
||||
pass
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_second += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
|
||||
def report_ul_bytes(self, num_bytes):
|
||||
self.ul_bytes_this_second += num_bytes
|
||||
self.total_ul_bytes += num_bytes
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""This class ensures that upload and download rates don't exceed specified maximums"""
|
||||
|
||||
#implements(IRateLimiter)
|
||||
|
||||
#called by main application
|
||||
|
||||
def __init__(self, max_dl_bytes=None, max_ul_bytes=None):
|
||||
self.max_dl_bytes = max_dl_bytes
|
||||
self.max_ul_bytes = max_ul_bytes
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.total_dl_bytes = 0
|
||||
self.total_ul_bytes = 0
|
||||
self.tick_call = None
|
||||
self.tick_interval = 0.1
|
||||
|
||||
self.dl_throttled = False
|
||||
self.ul_throttled = False
|
||||
|
||||
self.protocols = []
|
||||
|
||||
def start(self):
|
||||
log.info("Starting rate limiter.")
|
||||
self.tick_call = task.LoopingCall(self.tick)
|
||||
self.tick_call.start(self.tick_interval)
|
||||
|
||||
def tick(self):
|
||||
self.dl_bytes_this_interval = 0
|
||||
self.ul_bytes_this_interval = 0
|
||||
self.unthrottle_dl()
|
||||
self.unthrottle_ul()
|
||||
|
||||
def stop(self):
|
||||
log.info("Stopping rate limiter.")
|
||||
if self.tick_call is not None:
|
||||
self.tick_call.stop()
|
||||
self.tick_call = None
|
||||
|
||||
def set_dl_limit(self, limit):
|
||||
self.max_dl_bytes = limit
|
||||
|
||||
def set_ul_limit(self, limit):
|
||||
self.max_ul_bytes = limit
|
||||
|
||||
#throttling
|
||||
|
||||
def check_dl(self):
|
||||
need_throttle = (self.max_dl_bytes is not None and
|
||||
self.dl_bytes_this_interval > self.max_dl_bytes * self.tick_interval)
|
||||
if need_throttle:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_dl)
|
||||
|
||||
def check_ul(self):
|
||||
need_throttle = (self.max_ul_bytes is not None and
|
||||
self.ul_bytes_this_interval > self.max_ul_bytes * self.tick_interval)
|
||||
if need_throttle:
|
||||
from twisted.internet import reactor
|
||||
reactor.callLater(0, self.throttle_ul)
|
||||
|
||||
def throttle_dl(self):
|
||||
if self.dl_throttled is False:
|
||||
for protocol in self.protocols:
|
||||
protocol.throttle_download()
|
||||
self.dl_throttled = True
|
||||
|
||||
def throttle_ul(self):
|
||||
if self.ul_throttled is False:
|
||||
for protocol in self.protocols:
|
||||
protocol.throttle_upload()
|
||||
self.ul_throttled = True
|
||||
|
||||
def unthrottle_dl(self):
|
||||
if self.dl_throttled is True:
|
||||
for protocol in self.protocols:
|
||||
protocol.unthrottle_download()
|
||||
self.dl_throttled = False
|
||||
|
||||
def unthrottle_ul(self):
|
||||
if self.ul_throttled is True:
|
||||
for protocol in self.protocols:
|
||||
protocol.unthrottle_upload()
|
||||
self.ul_throttled = False
|
||||
|
||||
#called by protocols
|
||||
|
||||
def report_dl_bytes(self, num_bytes):
|
||||
self.dl_bytes_this_interval += num_bytes
|
||||
self.total_dl_bytes += num_bytes
|
||||
self.check_dl()
|
||||
|
||||
def report_ul_bytes(self, num_bytes):
|
||||
self.ul_bytes_this_interval += num_bytes
|
||||
self.total_ul_bytes += num_bytes
|
||||
self.check_ul()
|
||||
|
||||
def register_protocol(self, protocol):
|
||||
if protocol not in self.protocols:
|
||||
self.protocols.append(protocol)
|
||||
if self.dl_throttled is True:
|
||||
protocol.throttle_download()
|
||||
if self.ul_throttled is True:
|
||||
protocol.throttle_upload()
|
||||
|
||||
def unregister_protocol(self, protocol):
|
||||
if protocol in self.protocols:
|
||||
self.protocols.remove(protocol)
|
||||
@@ -1,109 +0,0 @@
|
||||
import logging
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from twisted.internet import defer, threads, reactor
|
||||
|
||||
from lbrynet.blob import BlobFile
|
||||
from lbrynet.core.BlobManager import DiskBlobManager
|
||||
from lbrynet.core.RateLimiter import DummyRateLimiter
|
||||
from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.database.storage import SQLiteStorage
|
||||
from lbrynet.dht.peerfinder import DummyPeerFinder
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SinglePeerFinder(DummyPeerFinder):
|
||||
def __init__(self, peer):
|
||||
super().__init__()
|
||||
self.peer = peer
|
||||
|
||||
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=False):
|
||||
return defer.succeed([self.peer])
|
||||
|
||||
|
||||
class BlobCallback(BlobFile):
|
||||
def __init__(self, blob_dir, blob_hash, timeout):
|
||||
super().__init__(blob_dir, blob_hash)
|
||||
self.callback = defer.Deferred()
|
||||
reactor.callLater(timeout, self._cancel)
|
||||
|
||||
def _cancel(self):
|
||||
if not self.callback.called:
|
||||
self.callback.callback(False)
|
||||
|
||||
def save_verified_blob(self, writer):
|
||||
result = BlobFile.save_verified_blob(self, writer)
|
||||
if not self.callback.called:
|
||||
self.callback.callback(True)
|
||||
return result
|
||||
|
||||
|
||||
class SingleBlobDownloadManager:
|
||||
def __init__(self, blob):
|
||||
self.blob = blob
|
||||
|
||||
def needed_blobs(self):
|
||||
if self.blob.verified:
|
||||
return []
|
||||
else:
|
||||
return [self.blob]
|
||||
|
||||
def get_head_blob_hash(self):
|
||||
return self.blob.blob_hash
|
||||
|
||||
|
||||
class SinglePeerDownloader:
|
||||
def __init__(self):
|
||||
self._payment_rate_manager = OnlyFreePaymentsManager()
|
||||
self._rate_limiter = DummyRateLimiter()
|
||||
self._wallet = None
|
||||
self._blob_manager = None
|
||||
|
||||
def setup(self, wallet, blob_manager=None):
|
||||
if not self._wallet:
|
||||
self._wallet = wallet
|
||||
if not self._blob_manager:
|
||||
self._blob_manager = blob_manager
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_blob_from_peer(self, peer, timeout, blob_hash, blob_manager):
|
||||
log.debug("Try to download %s from %s", blob_hash, peer.host)
|
||||
blob_manager = blob_manager
|
||||
blob = BlobCallback(blob_manager.blob_dir, blob_hash, timeout)
|
||||
download_manager = SingleBlobDownloadManager(blob)
|
||||
peer_finder = SinglePeerFinder(peer)
|
||||
requester = BlobRequester(blob_manager, peer_finder, self._payment_rate_manager,
|
||||
self._wallet, download_manager)
|
||||
downloader = StandaloneBlobDownloader(blob_hash, blob_manager, peer_finder,
|
||||
self._rate_limiter, self._payment_rate_manager,
|
||||
self._wallet, timeout=timeout)
|
||||
info_exchanger = self._wallet.get_info_exchanger()
|
||||
connection_manager = ConnectionManager(downloader, self._rate_limiter, [requester],
|
||||
[info_exchanger])
|
||||
connection_manager.start()
|
||||
|
||||
result = yield blob.callback
|
||||
if not result:
|
||||
log.debug("Failed to downloaded %s from %s", blob_hash[:16], peer.host)
|
||||
yield connection_manager.stop()
|
||||
defer.returnValue(result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
|
||||
tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
|
||||
tmp_storage = SQLiteStorage(tmp_dir)
|
||||
yield tmp_storage.setup()
|
||||
tmp_blob_manager = DiskBlobManager(tmp_dir, tmp_storage)
|
||||
try:
|
||||
result = yield self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager)
|
||||
finally:
|
||||
yield tmp_blob_manager.stop()
|
||||
yield tmp_storage.stop()
|
||||
yield threads.deferToThread(shutil.rmtree, tmp_dir)
|
||||
defer.returnValue(result)
|
||||
@@ -1,152 +0,0 @@
|
||||
from zope.interface import implementer
|
||||
from decimal import Decimal
|
||||
from lbrynet import conf
|
||||
from lbrynet.interfaces import INegotiationStrategy
|
||||
from lbrynet.core.Offer import Offer
|
||||
from lbrynet.core.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
|
||||
|
||||
|
||||
def get_default_strategy(blob_tracker, **kwargs):
|
||||
return BasicAvailabilityWeightedStrategy(blob_tracker, **kwargs)
|
||||
|
||||
|
||||
class Strategy:
|
||||
"""
|
||||
Base for negotiation strategies
|
||||
"""
|
||||
implementer(INegotiationStrategy)
|
||||
|
||||
def __init__(self, price_model, max_rate, min_rate, is_generous=None):
|
||||
self.price_model = price_model
|
||||
self.is_generous = (
|
||||
is_generous if is_generous is not None else conf.settings['is_generous_host'])
|
||||
self.accepted_offers = {}
|
||||
self.pending_sent_offers = {}
|
||||
self.offers_sent = {}
|
||||
self.offers_received = {}
|
||||
self.max_rate = max_rate or Decimal(self.price_model.base_price * 50)
|
||||
self.min_rate = Decimal(min_rate)
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
return NotImplementedError()
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
return NotImplementedError()
|
||||
|
||||
def make_offer(self, peer, blobs):
|
||||
offer_count = self.offers_sent.get(peer, 0)
|
||||
self._add_offer_sent(peer)
|
||||
if peer in self.accepted_offers:
|
||||
# if there was a previous accepted offer, use that
|
||||
offer = self.accepted_offers[peer]
|
||||
if peer in self.pending_sent_offers:
|
||||
del self.pending_sent_offers[peer]
|
||||
elif offer_count == 0 and self.is_generous:
|
||||
# Try asking for it for free
|
||||
offer = Offer(Decimal(0.0))
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
else:
|
||||
rates = [self.price_model.calculate_price(blob) for blob in blobs]
|
||||
price = self._make_rate_offer(rates, offer_count)
|
||||
offer = Offer(price)
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
return offer
|
||||
|
||||
def respond_to_offer(self, offer, peer, blobs):
|
||||
offer_count = self.offers_received.get(peer, 0)
|
||||
self._add_offer_received(peer)
|
||||
rates = [self.price_model.calculate_price(blob) for blob in blobs]
|
||||
price = self._get_response_rate(rates, offer_count)
|
||||
if peer in self.accepted_offers:
|
||||
offer = self.accepted_offers[peer]
|
||||
elif offer.rate == 0.0 and offer_count == 0 and self.is_generous:
|
||||
# give blobs away for free by default on the first request
|
||||
offer.accept()
|
||||
self.accepted_offers.update({peer: offer})
|
||||
elif offer.rate >= price:
|
||||
offer.accept()
|
||||
self.accepted_offers.update({peer: offer})
|
||||
else:
|
||||
offer.reject()
|
||||
if peer in self.accepted_offers:
|
||||
del self.accepted_offers[peer]
|
||||
return offer
|
||||
|
||||
def update_accepted_offers(self, peer, offer):
|
||||
if not offer.is_accepted and peer in self.accepted_offers:
|
||||
del self.accepted_offers[peer]
|
||||
if offer.is_accepted:
|
||||
self.accepted_offers.update({peer: offer})
|
||||
self.pending_sent_offers.update({peer: offer})
|
||||
|
||||
def _add_offer_sent(self, peer):
|
||||
turn = self.offers_sent.get(peer, 0) + 1
|
||||
self.offers_sent.update({peer: turn})
|
||||
|
||||
def _add_offer_received(self, peer):
|
||||
turn = self.offers_received.get(peer, 0) + 1
|
||||
self.offers_received.update({peer: turn})
|
||||
|
||||
def _bounded_price(self, price):
|
||||
price_for_return = Decimal(min(self.max_rate, max(price, self.min_rate)))
|
||||
return price_for_return
|
||||
|
||||
|
||||
class BasicAvailabilityWeightedStrategy(Strategy):
|
||||
"""Basic strategy to target blob prices based on supply relative to mean supply
|
||||
|
||||
Discount price target with each incoming request, and raise it
|
||||
with each outgoing from the modeled price until the rate is
|
||||
accepted or a threshold is reached
|
||||
|
||||
"""
|
||||
implementer(INegotiationStrategy)
|
||||
|
||||
def __init__(self, blob_tracker, acceleration=1.25,
|
||||
deceleration=0.9, max_rate=None,
|
||||
min_rate=0.0,
|
||||
is_generous=None,
|
||||
base_price=0.0001, alpha=1.0):
|
||||
price_model = MeanAvailabilityWeightedPrice(
|
||||
blob_tracker, base_price=base_price, alpha=alpha)
|
||||
super().__init__(price_model, max_rate, min_rate, is_generous)
|
||||
self._acceleration = Decimal(acceleration) # rate of how quickly to ramp offer
|
||||
self._deceleration = Decimal(deceleration)
|
||||
|
||||
def _get_mean_rate(self, rates):
|
||||
mean_rate = Decimal(sum(rates)) / Decimal(max(len(rates), 1))
|
||||
return mean_rate
|
||||
|
||||
def _premium(self, rate, turn):
|
||||
return rate * (self._acceleration ** Decimal(turn))
|
||||
|
||||
def _discount(self, rate, turn):
|
||||
return rate * (self._deceleration ** Decimal(turn))
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
rate = self._get_mean_rate(rates)
|
||||
discounted = self._discount(rate, offer_count)
|
||||
rounded_price = round(discounted, 5)
|
||||
return self._bounded_price(rounded_price)
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
rate = self._get_mean_rate(rates)
|
||||
with_premium = self._premium(rate, offer_count)
|
||||
rounded_price = round(with_premium, 5)
|
||||
return self._bounded_price(rounded_price)
|
||||
|
||||
|
||||
class OnlyFreeStrategy(Strategy):
|
||||
implementer(INegotiationStrategy)
|
||||
def __init__(self, *args, **kwargs):
|
||||
price_model = ZeroPrice()
|
||||
super().__init__(price_model, 0.0, 0.0, True)
|
||||
|
||||
def _get_mean_rate(self, rates):
|
||||
return 0.0
|
||||
|
||||
def _get_response_rate(self, rates, offer_count):
|
||||
return 0.0
|
||||
|
||||
def _make_rate_offer(self, rates, offer_count):
|
||||
return 0.0
|
||||
@@ -1,472 +0,0 @@
|
||||
from binascii import unhexlify
|
||||
import string
|
||||
from collections import defaultdict
|
||||
import json
|
||||
import logging
|
||||
|
||||
from twisted.internet import threads, defer
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
|
||||
from lbrynet.core.HTTPBlobDownloader import HTTPBlobDownloader
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class JSONBytesEncoder(json.JSONEncoder):
|
||||
def default(self, obj): # pylint: disable=E0202
|
||||
if isinstance(obj, bytes):
|
||||
return obj.decode()
|
||||
return super().default(obj)
|
||||
|
||||
|
||||
class StreamDescriptorReader:
|
||||
"""Classes which derive from this class read a stream descriptor file return
|
||||
a dictionary containing the fields in the file"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def _get_raw_data(self):
|
||||
"""This method must be overridden by subclasses. It should return a deferred
|
||||
which fires with the raw data in the stream descriptor"""
|
||||
pass
|
||||
|
||||
def get_info(self):
|
||||
"""Return the fields contained in the file"""
|
||||
d = self._get_raw_data()
|
||||
d.addCallback(json.loads)
|
||||
return d
|
||||
|
||||
|
||||
class PlainStreamDescriptorReader(StreamDescriptorReader):
|
||||
"""Read a stream descriptor file which is not a blob but a regular file"""
|
||||
def __init__(self, stream_descriptor_filename):
|
||||
super().__init__()
|
||||
self.stream_descriptor_filename = stream_descriptor_filename
|
||||
|
||||
def _get_raw_data(self):
|
||||
|
||||
def get_data():
|
||||
with open(self.stream_descriptor_filename) as file_handle:
|
||||
raw_data = file_handle.read()
|
||||
return raw_data
|
||||
|
||||
return threads.deferToThread(get_data)
|
||||
|
||||
|
||||
class BlobStreamDescriptorReader(StreamDescriptorReader):
|
||||
"""Read a stream descriptor file which is a blob"""
|
||||
def __init__(self, blob):
|
||||
super().__init__()
|
||||
self.blob = blob
|
||||
|
||||
def _get_raw_data(self):
|
||||
|
||||
def get_data():
|
||||
f = self.blob.open_for_reading()
|
||||
if f is not None:
|
||||
raw_data = f.read()
|
||||
f.close()
|
||||
return raw_data
|
||||
else:
|
||||
raise ValueError("Could not open the blob for reading")
|
||||
|
||||
return threads.deferToThread(get_data)
|
||||
|
||||
|
||||
class StreamDescriptorWriter:
|
||||
"""Classes which derive from this class write fields from a dictionary
|
||||
of fields to a stream descriptor"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def create_descriptor(self, sd_info):
|
||||
return self._write_stream_descriptor(
|
||||
json.dumps(sd_info, sort_keys=True).encode()
|
||||
)
|
||||
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
"""This method must be overridden by subclasses to write raw data to
|
||||
the stream descriptor
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class PlainStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
def __init__(self, sd_file_name):
|
||||
super().__init__()
|
||||
self.sd_file_name = sd_file_name
|
||||
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
|
||||
def write_file():
|
||||
log.info("Writing the sd file to disk")
|
||||
with open(self.sd_file_name, 'w') as sd_file:
|
||||
sd_file.write(raw_data)
|
||||
return self.sd_file_name
|
||||
|
||||
return threads.deferToThread(write_file)
|
||||
|
||||
|
||||
class BlobStreamDescriptorWriter(StreamDescriptorWriter):
|
||||
def __init__(self, blob_manager):
|
||||
super().__init__()
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _write_stream_descriptor(self, raw_data):
|
||||
log.debug("Creating the new blob for the stream descriptor")
|
||||
blob_creator = self.blob_manager.get_blob_creator()
|
||||
blob_creator.write(raw_data)
|
||||
log.debug("Wrote the data to the new blob")
|
||||
sd_hash = yield blob_creator.close()
|
||||
yield self.blob_manager.creator_finished(blob_creator, should_announce=True)
|
||||
defer.returnValue(sd_hash)
|
||||
|
||||
|
||||
class StreamMetadata:
|
||||
FROM_BLOB = 1
|
||||
FROM_PLAIN = 2
|
||||
|
||||
def __init__(self, validator, options, factories):
|
||||
self.validator = validator
|
||||
self.options = options
|
||||
self.factories = factories
|
||||
self.metadata_source = None
|
||||
self.source_blob_hash = None
|
||||
self.source_file = None
|
||||
|
||||
|
||||
class StreamDescriptorIdentifier:
|
||||
"""Tries to determine the type of stream described by the stream descriptor using the
|
||||
'stream_type' field. Keeps a list of StreamDescriptorValidators and StreamDownloaderFactorys
|
||||
and returns the appropriate ones based on the type of the stream descriptor given
|
||||
"""
|
||||
def __init__(self):
|
||||
# {stream_type: IStreamDescriptorValidator}
|
||||
self._sd_info_validators = {}
|
||||
# {stream_type: IStreamOptions
|
||||
self._stream_options = {}
|
||||
# {stream_type: [IStreamDownloaderFactory]}
|
||||
self._stream_downloader_factories = defaultdict(list)
|
||||
|
||||
def add_stream_type(self, stream_type, sd_info_validator, stream_options):
|
||||
"""This is how the StreamDescriptorIdentifier learns about new types of stream descriptors.
|
||||
|
||||
There can only be one StreamDescriptorValidator for each type of stream.
|
||||
|
||||
@param stream_type: A string representing the type of stream
|
||||
descriptor. This must be unique to this stream descriptor.
|
||||
|
||||
@param sd_info_validator: A class implementing the
|
||||
IStreamDescriptorValidator interface. This class's
|
||||
constructor will be passed the raw metadata in the stream
|
||||
descriptor file and its 'validate' method will then be
|
||||
called. If the validation step fails, an exception will be
|
||||
thrown, preventing the stream descriptor from being
|
||||
further processed.
|
||||
|
||||
@param stream_options: A class implementing the IStreamOptions
|
||||
interface. This class's constructor will be passed the
|
||||
sd_info_validator object containing the raw metadata from
|
||||
the stream descriptor file.
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
self._sd_info_validators[stream_type] = sd_info_validator
|
||||
self._stream_options[stream_type] = stream_options
|
||||
|
||||
def add_stream_downloader_factory(self, stream_type, factory):
|
||||
"""Register a stream downloader factory with the StreamDescriptorIdentifier.
|
||||
|
||||
This is how the StreamDescriptorIdentifier determines what
|
||||
factories may be used to process different stream descriptor
|
||||
files. There must be at least one factory for each type of
|
||||
stream added via "add_stream_info_validator".
|
||||
|
||||
@param stream_type: A string representing the type of stream
|
||||
descriptor which the factory knows how to process.
|
||||
|
||||
@param factory: An object implementing the IStreamDownloaderFactory interface.
|
||||
|
||||
@return: None
|
||||
|
||||
"""
|
||||
self._stream_downloader_factories[stream_type].append(factory)
|
||||
|
||||
def _return_metadata(self, options_validator_factories, source_type, source):
|
||||
validator, options, factories = options_validator_factories
|
||||
m = StreamMetadata(validator, options, factories)
|
||||
m.metadata_source = source_type
|
||||
if source_type == StreamMetadata.FROM_BLOB:
|
||||
m.source_blob_hash = source
|
||||
if source_type == StreamMetadata.FROM_PLAIN:
|
||||
m.source_file = source
|
||||
return m
|
||||
|
||||
def get_metadata_for_sd_file(self, sd_path):
|
||||
sd_reader = PlainStreamDescriptorReader(sd_path)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_PLAIN, sd_path)
|
||||
return d
|
||||
|
||||
def get_metadata_for_sd_blob(self, sd_blob):
|
||||
sd_reader = BlobStreamDescriptorReader(sd_blob)
|
||||
d = sd_reader.get_info()
|
||||
d.addCallback(self._return_options_and_validator_and_factories)
|
||||
d.addCallback(self._return_metadata, StreamMetadata.FROM_BLOB, sd_blob.blob_hash)
|
||||
return d
|
||||
|
||||
def _get_factories(self, stream_type):
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_downloader_factories[stream_type]
|
||||
|
||||
def _get_validator(self, stream_type):
|
||||
if not stream_type in self._sd_info_validators:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._sd_info_validators[stream_type]
|
||||
|
||||
def _get_options(self, stream_type):
|
||||
if not stream_type in self._stream_downloader_factories:
|
||||
raise UnknownStreamTypeError(stream_type)
|
||||
return self._stream_options[stream_type]
|
||||
|
||||
def _return_options_and_validator_and_factories(self, sd_info):
|
||||
if not 'stream_type' in sd_info:
|
||||
raise InvalidStreamDescriptorError('No stream_type parameter in stream descriptor.')
|
||||
stream_type = sd_info['stream_type']
|
||||
validator = self._get_validator(stream_type)(sd_info)
|
||||
factories = [f for f in self._get_factories(stream_type) if f.can_download(validator)]
|
||||
|
||||
d = validator.validate()
|
||||
|
||||
def get_options():
|
||||
options = self._get_options(stream_type)
|
||||
return validator, options, factories
|
||||
|
||||
d.addCallback(lambda _: get_options())
|
||||
return d
|
||||
|
||||
|
||||
EncryptedFileStreamType = "lbryfile"
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def save_sd_info(blob_manager, sd_hash, sd_info):
|
||||
if not blob_manager.blobs.get(sd_hash) or not blob_manager.blobs[sd_hash].get_is_verified():
|
||||
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
|
||||
calculated_sd_hash = yield descriptor_writer.create_descriptor(sd_info)
|
||||
if calculated_sd_hash != sd_hash:
|
||||
raise InvalidStreamDescriptorError("%s does not match calculated %s" %
|
||||
(sd_hash, calculated_sd_hash))
|
||||
stream_hash = yield blob_manager.storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
if not stream_hash:
|
||||
log.debug("Saving info for %s", unhexlify(sd_info['stream_name']))
|
||||
stream_name = sd_info['stream_name']
|
||||
key = sd_info['key']
|
||||
stream_hash = sd_info['stream_hash']
|
||||
stream_blobs = sd_info['blobs']
|
||||
suggested_file_name = sd_info['suggested_file_name']
|
||||
yield blob_manager.storage.add_known_blobs(stream_blobs)
|
||||
yield blob_manager.storage.store_stream(
|
||||
stream_hash, sd_hash, stream_name, key, suggested_file_name, stream_blobs
|
||||
)
|
||||
defer.returnValue(stream_hash)
|
||||
|
||||
|
||||
def format_blobs(crypt_blob_infos):
|
||||
formatted_blobs = []
|
||||
for blob_info in crypt_blob_infos:
|
||||
blob = {}
|
||||
if blob_info.length != 0:
|
||||
blob['blob_hash'] = blob_info.blob_hash
|
||||
blob['blob_num'] = blob_info.blob_num
|
||||
blob['iv'] = blob_info.iv
|
||||
blob['length'] = blob_info.length
|
||||
formatted_blobs.append(blob)
|
||||
return formatted_blobs
|
||||
|
||||
|
||||
def format_sd_info(stream_type, stream_name, key, suggested_file_name, stream_hash, blobs):
|
||||
return {
|
||||
"stream_type": stream_type,
|
||||
"stream_name": stream_name,
|
||||
"key": key,
|
||||
"suggested_file_name": suggested_file_name,
|
||||
"stream_hash": stream_hash,
|
||||
"blobs": blobs
|
||||
}
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_sd_info(storage, stream_hash, include_blobs):
|
||||
"""
|
||||
Get an sd info dictionary from storage
|
||||
|
||||
:param storage: (SQLiteStorage) storage instance
|
||||
:param stream_hash: (str) stream hash
|
||||
:param include_blobs: (bool) include stream blob infos
|
||||
|
||||
:return: {
|
||||
"stream_type": "lbryfile",
|
||||
"stream_name": <hex encoded stream name>,
|
||||
"key": <stream key>,
|
||||
"suggested_file_name": <hex encoded suggested file name>,
|
||||
"stream_hash": <stream hash>,
|
||||
"blobs": [
|
||||
{
|
||||
"blob_hash": <head blob_hash>,
|
||||
"blob_num": 0,
|
||||
"iv": <iv>,
|
||||
"length": <head blob length>
|
||||
}, ...
|
||||
{
|
||||
"blob_num": <stream length>,
|
||||
"iv": <iv>,
|
||||
"length": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
stream_info = yield storage.get_stream_info(stream_hash)
|
||||
blobs = []
|
||||
if include_blobs:
|
||||
blobs = yield storage.get_blobs_for_stream(stream_hash)
|
||||
defer.returnValue(
|
||||
format_sd_info(
|
||||
EncryptedFileStreamType, stream_info[0], stream_info[1],
|
||||
stream_info[2], stream_hash, format_blobs(blobs)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def get_blob_hashsum(b):
|
||||
length = b['length']
|
||||
if length != 0:
|
||||
blob_hash = b['blob_hash']
|
||||
else:
|
||||
blob_hash = None
|
||||
blob_num = b['blob_num']
|
||||
iv = b['iv']
|
||||
blob_hashsum = get_lbry_hash_obj()
|
||||
if length != 0:
|
||||
blob_hashsum.update(blob_hash.encode())
|
||||
blob_hashsum.update(str(blob_num).encode())
|
||||
blob_hashsum.update(iv.encode())
|
||||
blob_hashsum.update(str(length).encode())
|
||||
return blob_hashsum.digest()
|
||||
|
||||
|
||||
def get_stream_hash(hex_stream_name, key, hex_suggested_file_name, blob_infos):
|
||||
h = get_lbry_hash_obj()
|
||||
h.update(hex_stream_name.encode())
|
||||
h.update(key.encode())
|
||||
h.update(hex_suggested_file_name.encode())
|
||||
blobs_hashsum = get_lbry_hash_obj()
|
||||
for blob in blob_infos:
|
||||
blobs_hashsum.update(get_blob_hashsum(blob))
|
||||
h.update(blobs_hashsum.digest())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def verify_hex(text, field_name):
|
||||
if not set(text).issubset(set(string.hexdigits)):
|
||||
raise InvalidStreamDescriptorError("%s is not a hex-encoded string" % field_name)
|
||||
|
||||
|
||||
def validate_descriptor(stream_info):
|
||||
try:
|
||||
hex_stream_name = stream_info['stream_name']
|
||||
key = stream_info['key']
|
||||
hex_suggested_file_name = stream_info['suggested_file_name']
|
||||
stream_hash = stream_info['stream_hash']
|
||||
blobs = stream_info['blobs']
|
||||
except KeyError as e:
|
||||
raise InvalidStreamDescriptorError("Missing '%s'" % (e.args[0]))
|
||||
if stream_info['blobs'][-1]['length'] != 0:
|
||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||
if any([False if blob_info['length'] > 0 else True for blob_info in stream_info['blobs'][:-1]]):
|
||||
raise InvalidStreamDescriptorError("Contains zero-length data blob")
|
||||
if 'blob_hash' in stream_info['blobs'][-1]:
|
||||
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
|
||||
|
||||
verify_hex(key, "key")
|
||||
verify_hex(hex_suggested_file_name, "suggested file name")
|
||||
verify_hex(stream_hash, "stream_hash")
|
||||
|
||||
calculated_stream_hash = get_stream_hash(
|
||||
hex_stream_name, key, hex_suggested_file_name, blobs
|
||||
)
|
||||
if calculated_stream_hash != stream_hash:
|
||||
raise InvalidStreamDescriptorError("Stream hash does not match stream metadata")
|
||||
return True
|
||||
|
||||
|
||||
class EncryptedFileStreamDescriptorValidator:
|
||||
def __init__(self, raw_info):
|
||||
self.raw_info = raw_info
|
||||
|
||||
def validate(self):
|
||||
return defer.succeed(validate_descriptor(self.raw_info))
|
||||
|
||||
def info_to_show(self):
|
||||
info = []
|
||||
info.append(("stream_name", unhexlify(self.raw_info.get("stream_name"))))
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
info.append(("stream_size", str(self.get_length_of_stream())))
|
||||
suggested_file_name = self.raw_info.get("suggested_file_name", None)
|
||||
if suggested_file_name is not None:
|
||||
suggested_file_name = unhexlify(suggested_file_name)
|
||||
info.append(("suggested_file_name", suggested_file_name))
|
||||
return info
|
||||
|
||||
def get_length_of_stream(self):
|
||||
size_so_far = 0
|
||||
for blob_info in self.raw_info.get("blobs", []):
|
||||
size_so_far += int(blob_info['length'])
|
||||
return size_so_far
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_sd_blob(blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet, timeout=None,
|
||||
download_mirrors=None):
|
||||
"""
|
||||
Downloads a single blob from the network
|
||||
|
||||
@param session:
|
||||
|
||||
@param blob_hash:
|
||||
|
||||
@param payment_rate_manager:
|
||||
|
||||
@return: An object of type HashBlob
|
||||
"""
|
||||
|
||||
downloader = StandaloneBlobDownloader(blob_hash,
|
||||
blob_manager,
|
||||
peer_finder,
|
||||
rate_limiter,
|
||||
payment_rate_manager,
|
||||
wallet,
|
||||
timeout)
|
||||
mirror = HTTPBlobDownloader(blob_manager, [blob_hash], download_mirrors or [], sd_hashes=[blob_hash], retry=False)
|
||||
mirror.start()
|
||||
sd_blob = yield downloader.download()
|
||||
mirror.stop()
|
||||
sd_reader = BlobStreamDescriptorReader(sd_blob)
|
||||
sd_info = yield sd_reader.get_info()
|
||||
try:
|
||||
validate_descriptor(sd_info)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
yield blob_manager.delete_blobs([blob_hash])
|
||||
raise err
|
||||
raw_sd = yield sd_reader._get_raw_data()
|
||||
yield blob_manager.storage.add_known_blob(blob_hash, len(raw_sd))
|
||||
yield save_sd_info(blob_manager, sd_blob.blob_hash, sd_info)
|
||||
defer.returnValue(sd_blob)
|
||||
@@ -1,9 +0,0 @@
|
||||
"""
|
||||
Classes and functions which can be used by any application wishing to make use of the LBRY network.
|
||||
|
||||
This includes classes for connecting to other peers and downloading blobs from them, listening for
|
||||
connections from peers and responding to their requests, managing locally stored blobs, sending
|
||||
and receiving payments, and locating peers in the DHT.
|
||||
"""
|
||||
|
||||
from lbrynet import custom_logger
|
||||
@@ -1,81 +0,0 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger()
|
||||
|
||||
MIN_DELAY = 0.0
|
||||
MAX_DELAY = 0.01
|
||||
DELAY_INCREMENT = 0.0001
|
||||
QUEUE_SIZE_THRESHOLD = 100
|
||||
|
||||
|
||||
class CallLaterManager:
|
||||
def __init__(self, callLater):
|
||||
"""
|
||||
:param callLater: (IReactorTime.callLater)
|
||||
"""
|
||||
|
||||
self._callLater = callLater
|
||||
self._pendingCallLaters = []
|
||||
self._delay = MIN_DELAY
|
||||
|
||||
def get_min_delay(self):
|
||||
self._pendingCallLaters = [cl for cl in self._pendingCallLaters if cl.active()]
|
||||
queue_size = len(self._pendingCallLaters)
|
||||
if queue_size > QUEUE_SIZE_THRESHOLD:
|
||||
self._delay = min((self._delay + DELAY_INCREMENT), MAX_DELAY)
|
||||
else:
|
||||
self._delay = max((self._delay - 2.0 * DELAY_INCREMENT), MIN_DELAY)
|
||||
return self._delay
|
||||
|
||||
def _cancel(self, call_later):
|
||||
"""
|
||||
:param call_later: DelayedCall
|
||||
:return: (callable) canceller function
|
||||
"""
|
||||
|
||||
def cancel(reason=None):
|
||||
"""
|
||||
:param reason: reason for cancellation, this is returned after cancelling the DelayedCall
|
||||
:return: reason
|
||||
"""
|
||||
|
||||
if call_later.active():
|
||||
call_later.cancel()
|
||||
if call_later in self._pendingCallLaters:
|
||||
self._pendingCallLaters.remove(call_later)
|
||||
return reason
|
||||
return cancel
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
Cancel any callLaters that are still running
|
||||
"""
|
||||
|
||||
from twisted.internet import defer
|
||||
while self._pendingCallLaters:
|
||||
canceller = self._cancel(self._pendingCallLaters[0])
|
||||
try:
|
||||
canceller()
|
||||
except (defer.CancelledError, defer.AlreadyCalledError, ValueError):
|
||||
pass
|
||||
|
||||
def call_later(self, when, what, *args, **kwargs):
|
||||
"""
|
||||
Schedule a call later and get a canceller callback function
|
||||
|
||||
:param when: (float) delay in seconds
|
||||
:param what: (callable)
|
||||
:param args: (*tuple) args to be passed to the callable
|
||||
:param kwargs: (**dict) kwargs to be passed to the callable
|
||||
|
||||
:return: (tuple) twisted.internet.base.DelayedCall object, canceller function
|
||||
"""
|
||||
|
||||
call_later = self._callLater(when, what, *args, **kwargs)
|
||||
canceller = self._cancel(call_later)
|
||||
self._pendingCallLaters.append(call_later)
|
||||
return call_later, canceller
|
||||
|
||||
def call_soon(self, what, *args, **kwargs):
|
||||
delay = self.get_min_delay()
|
||||
return self.call_later(delay, what, *args, **kwargs)
|
||||
@@ -1,590 +0,0 @@
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from decimal import Decimal
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet.error import ConnectionAborted
|
||||
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError
|
||||
from lbrynet.core.Error import InvalidResponseError, RequestCanceledError, NoResponseError
|
||||
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
|
||||
from lbrynet.core.client.ClientRequest import ClientRequest, ClientBlobRequest
|
||||
from lbrynet.core.Offer import Offer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_points(num_bytes, rate):
|
||||
if isinstance(rate, float):
|
||||
return 1.0 * num_bytes * rate / 2**20
|
||||
elif isinstance(rate, Decimal):
|
||||
return 1.0 * num_bytes * float(rate) / 2**20
|
||||
else:
|
||||
raise Exception("Unknown rate type")
|
||||
|
||||
|
||||
def cache(fn):
|
||||
"""Caches the function call for each instance"""
|
||||
attr = '__{}_value'.format(fn.__name__)
|
||||
|
||||
def helper(self):
|
||||
if not hasattr(self, attr):
|
||||
value = fn(self)
|
||||
setattr(self, attr, value)
|
||||
return getattr(self, attr)
|
||||
return helper
|
||||
|
||||
|
||||
class BlobRequester:
|
||||
#implements(IRequestCreator)
|
||||
|
||||
def __init__(self, blob_manager, peer_finder, payment_rate_manager, wallet, download_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_finder = peer_finder
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self._download_manager = download_manager
|
||||
self._peers = defaultdict(int) # {Peer: score}
|
||||
self._available_blobs = defaultdict(list) # {Peer: [blob_hash]}
|
||||
self._unavailable_blobs = defaultdict(list) # {Peer: [blob_hash]}}
|
||||
self._protocol_prices = {} # {ClientProtocol: price}
|
||||
self._protocol_offers = {}
|
||||
self._price_disagreements = [] # [Peer]
|
||||
self._protocol_tries = {}
|
||||
self._maxed_out_peers = []
|
||||
self._incompatible_peers = []
|
||||
|
||||
######## IRequestCreator #########
|
||||
def send_next_request(self, peer, protocol):
|
||||
"""Makes an availability request, download request and price request"""
|
||||
if not self.should_send_next_request(peer):
|
||||
return defer.succeed(False)
|
||||
return self._send_next_request(peer, protocol)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_new_peers_for_head_blob(self):
|
||||
""" look for peers for the head blob """
|
||||
head_blob_hash = self._download_manager.get_head_blob_hash()
|
||||
peers = yield self._find_peers_for_hash(head_blob_hash)
|
||||
defer.returnValue(peers)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_new_peers_for_next_unavailable(self):
|
||||
"""
|
||||
Look for peers for the next unavailable blob, if we have
|
||||
all blobs, return an empty list
|
||||
"""
|
||||
blob_hash = yield self._get_hash_for_peer_search()
|
||||
if blob_hash is None:
|
||||
defer.returnValue([])
|
||||
peers = yield self._find_peers_for_hash(blob_hash)
|
||||
defer.returnValue(peers)
|
||||
|
||||
######### internal calls #########
|
||||
def should_send_next_request(self, peer):
|
||||
return (
|
||||
self._blobs_to_download() and
|
||||
self._should_send_request_to(peer)
|
||||
)
|
||||
|
||||
def _send_next_request(self, peer, protocol):
|
||||
log.debug('Sending a blob request for %s and %s', peer, protocol)
|
||||
availability = AvailabilityRequest(self, peer, protocol, self.payment_rate_manager)
|
||||
head_blob_hash = self._download_manager.get_head_blob_hash()
|
||||
download = DownloadRequest(self, peer, protocol, self.payment_rate_manager,
|
||||
self.wallet, head_blob_hash)
|
||||
price = PriceRequest(self, peer, protocol, self.payment_rate_manager)
|
||||
|
||||
sent_request = False
|
||||
if availability.can_make_request():
|
||||
availability.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
if price.can_make_request():
|
||||
# TODO: document why a PriceRequest is only made if an
|
||||
# Availability or Download request was made
|
||||
price.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
if download.can_make_request():
|
||||
try:
|
||||
download.make_request_and_handle_response()
|
||||
sent_request = True
|
||||
except InsufficientFundsError as err:
|
||||
return defer.fail(err)
|
||||
|
||||
return defer.succeed(sent_request)
|
||||
|
||||
def _get_hash_for_peer_search(self):
|
||||
"""
|
||||
Get next unavailable hash for blob,
|
||||
returns None if there is nothing left to download
|
||||
"""
|
||||
r = None
|
||||
blobs_to_download = self._blobs_to_download()
|
||||
if blobs_to_download:
|
||||
blobs_without_sources = self._blobs_without_sources()
|
||||
if not blobs_without_sources:
|
||||
blob_hash = blobs_to_download[0].blob_hash
|
||||
else:
|
||||
blob_hash = blobs_without_sources[0].blob_hash
|
||||
r = blob_hash
|
||||
log.debug("Blob requester peer search response: %s", str(r))
|
||||
return defer.succeed(r)
|
||||
|
||||
def _find_peers_for_hash(self, h):
|
||||
d = self.peer_finder.find_peers_for_blob(h, filter_self=True)
|
||||
|
||||
def choose_best_peers(peers):
|
||||
bad_peers = self._get_bad_peers()
|
||||
without_bad_peers = [p for p in peers if not p in bad_peers]
|
||||
without_maxed_out_peers = [
|
||||
p for p in without_bad_peers if p not in self._maxed_out_peers]
|
||||
return without_maxed_out_peers
|
||||
|
||||
d.addCallback(choose_best_peers)
|
||||
|
||||
def lookup_failed(err):
|
||||
log.error("An error occurred looking up peers for a hash: %s", err.getTraceback())
|
||||
return []
|
||||
|
||||
d.addErrback(lookup_failed)
|
||||
return d
|
||||
|
||||
def _should_send_request_to(self, peer):
|
||||
if self._peers[peer] < -5.0:
|
||||
return False
|
||||
if peer in self._price_disagreements:
|
||||
return False
|
||||
if peer in self._incompatible_peers:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _get_bad_peers(self):
|
||||
return [p for p in self._peers.keys() if not self._should_send_request_to(p)]
|
||||
|
||||
def _hash_available(self, blob_hash):
|
||||
for peer in self._available_blobs:
|
||||
if blob_hash in self._available_blobs[peer]:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _hash_available_on(self, blob_hash, peer):
|
||||
if blob_hash in self._available_blobs[peer]:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _blobs_to_download(self):
|
||||
needed_blobs = self._download_manager.needed_blobs()
|
||||
return sorted(needed_blobs, key=lambda b: b.is_downloading())
|
||||
|
||||
def _blobs_without_sources(self):
|
||||
return [
|
||||
b for b in self._download_manager.needed_blobs()
|
||||
if not self._hash_available(b.blob_hash)
|
||||
]
|
||||
|
||||
def _price_settled(self, protocol):
|
||||
if protocol in self._protocol_prices:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _update_local_score(self, peer, amount):
|
||||
self._peers[peer] += amount
|
||||
|
||||
|
||||
class RequestHelper:
|
||||
def __init__(self, requestor, peer, protocol, payment_rate_manager):
|
||||
self.requestor = requestor
|
||||
self.peer = peer
|
||||
self.protocol = protocol
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
|
||||
@property
|
||||
def protocol_prices(self):
|
||||
return self.requestor._protocol_prices
|
||||
|
||||
@property
|
||||
def protocol_offers(self):
|
||||
return self.requestor._protocol_offers
|
||||
|
||||
@property
|
||||
def available_blobs(self):
|
||||
return self.requestor._available_blobs[self.peer]
|
||||
|
||||
@property
|
||||
def unavailable_blobs(self):
|
||||
return self.requestor._unavailable_blobs[self.peer]
|
||||
|
||||
@property
|
||||
def maxed_out_peers(self):
|
||||
return self.requestor._maxed_out_peers
|
||||
|
||||
def update_local_score(self, score):
|
||||
self.requestor._update_local_score(self.peer, score)
|
||||
|
||||
def _request_failed(self, reason, request_type):
|
||||
if reason.check(DownloadCanceledError, RequestCanceledError, ConnectionAborted,
|
||||
ConnectionClosedBeforeResponseError, ValueError):
|
||||
return
|
||||
if reason.check(NoResponseError):
|
||||
self.requestor._incompatible_peers.append(self.peer)
|
||||
log.warning("A request of type '%s' failed. Reason: %s, Error type: %s",
|
||||
request_type, reason.getErrorMessage(), reason.type)
|
||||
self.update_local_score(-10.0)
|
||||
if isinstance(reason, (InvalidResponseError, NoResponseError)):
|
||||
self.peer.update_score(-10.0)
|
||||
else:
|
||||
self.peer.update_score(-2.0)
|
||||
return reason
|
||||
|
||||
def get_rate(self):
|
||||
if self.payment_rate_manager.price_limit_reached(self.peer):
|
||||
if self.peer not in self.maxed_out_peers:
|
||||
self.maxed_out_peers.append(self.peer)
|
||||
return None
|
||||
rate = self.protocol_prices.get(self.protocol)
|
||||
if rate is None:
|
||||
if self.peer in self.payment_rate_manager.strategy.pending_sent_offers:
|
||||
pending = self.payment_rate_manager.strategy.pending_sent_offers[self.peer]
|
||||
if not pending.is_too_low and not pending.is_accepted:
|
||||
return pending.rate
|
||||
rate = self.payment_rate_manager.get_rate_blob_data(self.peer, self.available_blobs)
|
||||
return rate
|
||||
|
||||
|
||||
def _handle_incoming_blob(response_dict, peer, request):
|
||||
if request.response_identifier not in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
if not isinstance(response_dict[request.response_identifier], dict):
|
||||
return InvalidResponseError("response not a dict. got %s" %
|
||||
type(response_dict[request.response_identifier]))
|
||||
response = response_dict[request.response_identifier]
|
||||
if 'error' in response:
|
||||
# This means we're not getting our blob for some reason
|
||||
if response['error'] == "RATE_UNSET":
|
||||
# Stop the download with an error that won't penalize the peer
|
||||
request.cancel(PriceDisagreementError())
|
||||
else:
|
||||
# The peer has done something bad so we should get out of here
|
||||
return InvalidResponseError("Got an unknown error from the peer: %s" %
|
||||
(response['error'],))
|
||||
else:
|
||||
if 'blob_hash' not in response:
|
||||
return InvalidResponseError("Missing the required field 'blob_hash'")
|
||||
if not response['blob_hash'] == request.request_dict['requested_blob']:
|
||||
return InvalidResponseError(
|
||||
"Incoming blob does not match expected. Incoming: %s. Expected: %s" %
|
||||
(response['blob_hash'], request.request_dict['requested_blob'])
|
||||
)
|
||||
if 'length' not in response:
|
||||
return InvalidResponseError("Missing the required field 'length'")
|
||||
if not request.blob.set_length(response['length']):
|
||||
return InvalidResponseError("Could not set the length of the blob")
|
||||
return True
|
||||
|
||||
|
||||
def _handle_download_error(err, peer, blob_to_download):
|
||||
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError,
|
||||
ConnectionClosedBeforeResponseError):
|
||||
log.warning("An error occurred while downloading %s from %s. Error: %s",
|
||||
blob_to_download.blob_hash, str(peer), err.getTraceback())
|
||||
if err.check(PriceDisagreementError):
|
||||
# Don't kill the whole connection just because a price couldn't be agreed upon.
|
||||
# Other information might be desired by other request creators at a better rate.
|
||||
return True
|
||||
return err
|
||||
|
||||
|
||||
class AvailabilityRequest(RequestHelper):
|
||||
"""Ask a peer what blobs it has available.
|
||||
|
||||
Results are saved in `_available_blobs` and `_unavailable_blobs`
|
||||
on the parent BlobRequester.
|
||||
"""
|
||||
def can_make_request(self):
|
||||
return self.get_top_needed_blobs()
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_request()
|
||||
self._handle_request(request)
|
||||
|
||||
def _get_request(self):
|
||||
to_request = self.get_top_needed_blobs()
|
||||
if not to_request:
|
||||
raise Exception('Unable to make a request without available blobs')
|
||||
return self._make_request(to_request)
|
||||
|
||||
@cache
|
||||
def get_top_needed_blobs(self, limit=20):
|
||||
all_needed = [
|
||||
b.blob_hash for b in self.requestor._blobs_to_download()
|
||||
if not self.is_available(b)
|
||||
]
|
||||
# sort them so that the peer will be asked first for blobs it
|
||||
# hasn't said it doesn't have
|
||||
sorted_needed = sorted(
|
||||
all_needed,
|
||||
key=lambda b: b in self.unavailable_blobs
|
||||
)
|
||||
return sorted_needed[:limit]
|
||||
|
||||
def is_available(self, blob):
|
||||
return blob.blob_hash in self.available_blobs
|
||||
|
||||
def _make_request(self, to_request):
|
||||
log.debug('Requesting blobs: %s', to_request)
|
||||
r_dict = {'requested_blobs': to_request}
|
||||
response_identifier = 'available_blobs'
|
||||
request = ClientRequest(r_dict, response_identifier)
|
||||
return request
|
||||
|
||||
def _handle_request(self, a_r):
|
||||
log.debug('making an availability request')
|
||||
d1 = self.protocol.add_request(a_r)
|
||||
d1.addCallback(self._handle_availability, a_r)
|
||||
d1.addErrback(self._request_failed, "availability request")
|
||||
|
||||
def _handle_availability(self, response_dict, request):
|
||||
assert request.response_identifier == 'available_blobs'
|
||||
if 'available_blobs' not in response_dict:
|
||||
raise InvalidResponseError("response identifier not in response")
|
||||
log.debug("Received a response to the availability request")
|
||||
# save available blobs
|
||||
blob_hashes = response_dict['available_blobs']
|
||||
if not blob_hashes:
|
||||
# should not send any more requests as it doesn't have any blob we need
|
||||
self.update_local_score(-10.0)
|
||||
return True
|
||||
for blob_hash in blob_hashes:
|
||||
if blob_hash in request.request_dict['requested_blobs']:
|
||||
self.process_available_blob_hash(blob_hash, request)
|
||||
# everything left in the request is missing
|
||||
for blob_hash in request.request_dict['requested_blobs']:
|
||||
self.unavailable_blobs.append(blob_hash)
|
||||
return True
|
||||
|
||||
def process_available_blob_hash(self, blob_hash, request):
|
||||
log.debug("The server has indicated it has the following blob available: %s", blob_hash)
|
||||
self.available_blobs.append(blob_hash)
|
||||
self.remove_from_unavailable_blobs(blob_hash)
|
||||
request.request_dict['requested_blobs'].remove(blob_hash)
|
||||
|
||||
def remove_from_unavailable_blobs(self, blob_hash):
|
||||
if blob_hash in self.unavailable_blobs:
|
||||
self.unavailable_blobs.remove(blob_hash)
|
||||
|
||||
|
||||
class PriceRequest(RequestHelper):
|
||||
"""Ask a peer if a certain price is acceptable"""
|
||||
def can_make_request(self):
|
||||
if len(self.available_blobs) and self.protocol not in self.protocol_prices:
|
||||
return self.get_rate() is not None
|
||||
return False
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_price_request()
|
||||
self._handle_price_request(request)
|
||||
|
||||
def _get_price_request(self):
|
||||
rate = self.get_rate()
|
||||
if rate is None:
|
||||
log.debug("No blobs to request from %s", self.peer)
|
||||
raise Exception('Cannot make a price request without a payment rate')
|
||||
log.debug("Offer rate %s to %s for %i blobs", rate, self.peer, len(self.available_blobs))
|
||||
|
||||
request_dict = {'blob_data_payment_rate': rate}
|
||||
assert self.protocol not in self.protocol_offers
|
||||
self.protocol_offers[self.protocol] = rate
|
||||
return ClientRequest(request_dict, 'blob_data_payment_rate')
|
||||
|
||||
def _handle_price_request(self, price_request):
|
||||
d = self.protocol.add_request(price_request)
|
||||
d.addCallback(self._handle_price_response, price_request)
|
||||
d.addErrback(self._request_failed, "price request")
|
||||
|
||||
def _handle_price_response(self, response_dict, request):
|
||||
assert request.response_identifier == 'blob_data_payment_rate'
|
||||
if 'blob_data_payment_rate' not in response_dict:
|
||||
return InvalidResponseError("response identifier not in response")
|
||||
offer_value = self.protocol_offers.pop(self.protocol)
|
||||
offer = Offer(offer_value)
|
||||
offer.handle(response_dict['blob_data_payment_rate'])
|
||||
self.payment_rate_manager.record_offer_reply(self.peer, offer)
|
||||
if offer.is_accepted:
|
||||
log.info("Offered rate %f/mb accepted by %s", offer.rate, self.peer.host)
|
||||
self.protocol_prices[self.protocol] = offer.rate
|
||||
return True
|
||||
elif offer.is_too_low:
|
||||
log.debug("Offered rate %f/mb rejected by %s", offer.rate, self.peer.host)
|
||||
return not self.payment_rate_manager.price_limit_reached(self.peer)
|
||||
else:
|
||||
log.warning("Price disagreement")
|
||||
self.requestor._price_disagreements.append(self.peer)
|
||||
return False
|
||||
|
||||
|
||||
class DownloadRequest(RequestHelper):
|
||||
"""Choose a blob and download it from a peer and also pay the peer for the data."""
|
||||
def __init__(self, requester, peer, protocol, payment_rate_manager, wallet, head_blob_hash):
|
||||
super().__init__(requester, peer, protocol, payment_rate_manager)
|
||||
self.wallet = wallet
|
||||
self.head_blob_hash = head_blob_hash
|
||||
|
||||
def can_make_request(self):
|
||||
if self.protocol in self.protocol_prices:
|
||||
return self.get_blob_details()
|
||||
return False
|
||||
|
||||
def make_request_and_handle_response(self):
|
||||
request = self._get_request()
|
||||
self._handle_download_request(request)
|
||||
|
||||
def _get_request(self):
|
||||
blob_details = self.get_blob_details()
|
||||
if not blob_details:
|
||||
raise Exception('No blobs available to download')
|
||||
return self._make_request(blob_details)
|
||||
|
||||
@cache
|
||||
def get_blob_details(self):
|
||||
"""Open a blob for writing and return the details.
|
||||
|
||||
If no blob can be opened, returns None.
|
||||
"""
|
||||
to_download = self.get_available_blobs()
|
||||
return self.find_blob(to_download)
|
||||
|
||||
def get_available_blobs(self):
|
||||
available_blobs = [
|
||||
b for b in self.requestor._blobs_to_download()
|
||||
if self.requestor._hash_available_on(b.blob_hash, self.peer)
|
||||
]
|
||||
log.debug('available blobs: %s', available_blobs)
|
||||
return available_blobs
|
||||
|
||||
def find_blob(self, to_download):
|
||||
"""Return the first blob in `to_download` that is successfully opened for write."""
|
||||
for blob in to_download:
|
||||
if blob.get_is_verified():
|
||||
log.debug('Skipping blob %s as its already validated', blob)
|
||||
continue
|
||||
writer, d = blob.open_for_writing(self.peer)
|
||||
if d is not None:
|
||||
return BlobDownloadDetails(blob, d, writer.write, writer.close, self.peer)
|
||||
log.warning('Skipping blob %s as there was an issue opening it for writing', blob)
|
||||
return None
|
||||
|
||||
def _make_request(self, blob_details):
|
||||
blob = blob_details.blob
|
||||
request = ClientBlobRequest(
|
||||
{'requested_blob': blob.blob_hash},
|
||||
'incoming_blob',
|
||||
blob_details.counting_write_func,
|
||||
blob_details.deferred,
|
||||
blob_details.cancel_func,
|
||||
blob
|
||||
)
|
||||
log.info("Requesting blob %s from %s", blob.blob_hash, self.peer)
|
||||
return request
|
||||
|
||||
def _handle_download_request(self, client_blob_request):
|
||||
reserved_points = self.reserve_funds_or_cancel(client_blob_request)
|
||||
self.add_callbacks_to_download_request(client_blob_request, reserved_points)
|
||||
self.create_add_blob_request(client_blob_request)
|
||||
|
||||
def reserve_funds_or_cancel(self, client_blob_request):
|
||||
reserved_points = self._reserve_points(client_blob_request.max_pay_units)
|
||||
if reserved_points is not None:
|
||||
return reserved_points
|
||||
client_blob_request.cancel(InsufficientFundsError())
|
||||
client_blob_request.finished_deferred.addErrback(lambda _: True)
|
||||
raise InsufficientFundsError()
|
||||
|
||||
def add_callbacks_to_download_request(self, client_blob_request, reserved_points):
|
||||
# Note: The following three callbacks will be called when the blob has been
|
||||
# fully downloaded or canceled
|
||||
client_blob_request.finished_deferred.addCallbacks(
|
||||
self._download_succeeded,
|
||||
self._download_failed,
|
||||
callbackArgs=(client_blob_request.blob,),
|
||||
)
|
||||
client_blob_request.finished_deferred.addBoth(self._pay_or_cancel_payment,
|
||||
reserved_points, client_blob_request.blob)
|
||||
client_blob_request.finished_deferred.addErrback(_handle_download_error, self.peer,
|
||||
client_blob_request.blob)
|
||||
|
||||
def _pay_or_cancel_payment(self, arg, reserved_points, blob):
|
||||
if self._can_pay_peer(blob, arg):
|
||||
self._pay_peer(blob.length, reserved_points)
|
||||
else:
|
||||
self._cancel_points(reserved_points)
|
||||
return arg
|
||||
|
||||
def _can_pay_peer(self, blob, arg):
|
||||
return (
|
||||
blob.length != 0 and
|
||||
(not isinstance(arg, Failure) or arg.check(DownloadCanceledError))
|
||||
)
|
||||
|
||||
def _pay_peer(self, num_bytes, reserved_points):
|
||||
assert num_bytes != 0
|
||||
rate = self.get_rate()
|
||||
point_amount = get_points(num_bytes, rate)
|
||||
self.wallet.send_points(reserved_points, point_amount)
|
||||
self.payment_rate_manager.record_points_paid(point_amount)
|
||||
|
||||
def _cancel_points(self, reserved_points):
|
||||
self.wallet.cancel_point_reservation(reserved_points)
|
||||
|
||||
def create_add_blob_request(self, client_blob_request):
|
||||
d = self.protocol.add_blob_request(client_blob_request)
|
||||
# Note: The following two callbacks will be called as soon as the peer sends its
|
||||
# response, which will be before the blob has finished downloading, but may be
|
||||
# after the blob has been canceled. For example,
|
||||
# 1) client sends request to Peer A
|
||||
# 2) the blob is finished downloading from peer B, and therefore this one is canceled
|
||||
# 3) client receives response from Peer A
|
||||
# Therefore, these callbacks shouldn't rely on there being a blob about to be
|
||||
# downloaded.
|
||||
d.addCallback(_handle_incoming_blob, self.peer, client_blob_request)
|
||||
d.addErrback(self._request_failed, "download request")
|
||||
|
||||
def _reserve_points(self, num_bytes):
|
||||
# jobevers: there was an assertion here, but I don't think it
|
||||
# was a valid assertion to make. It is possible for a rate to
|
||||
# not yet been set for this protocol or for it to have been
|
||||
# removed so instead I switched it to check if a rate has been set
|
||||
# and calculate it if it has not
|
||||
rate = self.get_rate()
|
||||
points_to_reserve = get_points(num_bytes, rate)
|
||||
return self.wallet.reserve_points(self.peer, points_to_reserve)
|
||||
|
||||
def _download_succeeded(self, arg, blob):
|
||||
log.info("Blob %s has been successfully downloaded from %s", blob, self.peer)
|
||||
self.update_local_score(5.0)
|
||||
self.peer.update_stats('blobs_downloaded', 1)
|
||||
self.peer.update_score(5.0)
|
||||
should_announce = blob.blob_hash == self.head_blob_hash
|
||||
d = self.requestor.blob_manager.blob_completed(blob, should_announce=should_announce)
|
||||
d.addCallback(lambda _: arg)
|
||||
return d
|
||||
|
||||
def _download_failed(self, reason):
|
||||
if not reason.check(DownloadCanceledError, PriceDisagreementError):
|
||||
self.update_local_score(-10.0)
|
||||
return reason
|
||||
|
||||
|
||||
class BlobDownloadDetails:
|
||||
"""Contains the information needed to make a ClientBlobRequest from an open blob"""
|
||||
def __init__(self, blob, deferred, write_func, cancel_func, peer):
|
||||
self.blob = blob
|
||||
self.deferred = deferred
|
||||
self.write_func = write_func
|
||||
self.cancel_func = cancel_func
|
||||
self.peer = peer
|
||||
|
||||
def counting_write_func(self, data):
|
||||
self.peer.update_stats('blob_bytes_downloaded', len(data))
|
||||
return self.write_func(data)
|
||||
@@ -1,284 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
from twisted.internet import error, defer
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.protocols.policies import TimeoutMixin
|
||||
from twisted.python import failure
|
||||
from lbrynet import conf
|
||||
from lbrynet.core import utils
|
||||
from lbrynet.core.Error import ConnectionClosedBeforeResponseError, NoResponseError
|
||||
from lbrynet.core.Error import DownloadCanceledError, MisbehavingPeerError
|
||||
from lbrynet.core.Error import RequestCanceledError
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def encode_decimal(obj):
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
raise TypeError(repr(obj) + " is not JSON serializable")
|
||||
|
||||
|
||||
class ClientProtocol(Protocol, TimeoutMixin):
|
||||
#implements(IRequestSender, IRateLimited)
|
||||
######### Protocol #########
|
||||
PROTOCOL_TIMEOUT = 30
|
||||
|
||||
def connectionMade(self):
|
||||
log.debug("Connection made to %s", self.factory.peer)
|
||||
self._connection_manager = self.factory.connection_manager
|
||||
self._rate_limiter = self.factory.rate_limiter
|
||||
self.peer = self.factory.peer
|
||||
self._response_deferreds = {}
|
||||
self._response_buff = b''
|
||||
self._downloading_blob = False
|
||||
self._blob_download_request = None
|
||||
self._next_request = {}
|
||||
self.connection_closed = False
|
||||
self.connection_closing = False
|
||||
# This needs to be set for TimeoutMixin
|
||||
self.callLater = utils.call_later
|
||||
self.peer.report_up()
|
||||
|
||||
self._ask_for_request()
|
||||
|
||||
def dataReceived(self, data):
|
||||
log.debug("Received %d bytes from %s", len(data), self.peer)
|
||||
self.setTimeout(None)
|
||||
self._rate_limiter.report_dl_bytes(len(data))
|
||||
|
||||
if self._downloading_blob is True:
|
||||
self._blob_download_request.write(data)
|
||||
else:
|
||||
self._response_buff += data
|
||||
if len(self._response_buff) > conf.settings['MAX_RESPONSE_INFO_SIZE']:
|
||||
log.warning("Response is too large from %s. Size %s",
|
||||
self.peer, len(self._response_buff))
|
||||
self.transport.loseConnection()
|
||||
response, extra_data = self._get_valid_response(self._response_buff)
|
||||
if response is not None:
|
||||
self._response_buff = b''
|
||||
self._handle_response(response)
|
||||
if self._downloading_blob is True and len(extra_data) != 0:
|
||||
self._blob_download_request.write(extra_data)
|
||||
|
||||
def timeoutConnection(self):
|
||||
log.info("Connection timed out to %s", self.peer)
|
||||
self.peer.report_down()
|
||||
self.transport.abortConnection()
|
||||
|
||||
def connectionLost(self, reason=None):
|
||||
log.debug("Connection lost to %s: %s", self.peer, reason)
|
||||
self.setTimeout(None)
|
||||
self.connection_closed = True
|
||||
if reason is None or reason.check(error.ConnectionDone):
|
||||
err = failure.Failure(ConnectionClosedBeforeResponseError())
|
||||
else:
|
||||
err = reason
|
||||
for key, d in self._response_deferreds.items():
|
||||
d.errback(err)
|
||||
self._response_deferreds.clear()
|
||||
if self._blob_download_request is not None:
|
||||
self._blob_download_request.cancel(err)
|
||||
self.factory.connection_was_made_deferred.callback(True)
|
||||
|
||||
######### IRequestSender #########
|
||||
|
||||
def add_request(self, request):
|
||||
if request.response_identifier in self._response_deferreds:
|
||||
raise ValueError("There is already a request for that response active")
|
||||
self._next_request.update(request.request_dict)
|
||||
d = defer.Deferred()
|
||||
log.debug("Adding a request for %s. Request: %s", self.peer, request.request_dict)
|
||||
self._response_deferreds[request.response_identifier] = d
|
||||
return d
|
||||
|
||||
def add_blob_request(self, blob_request):
|
||||
if self._blob_download_request is None:
|
||||
d = self.add_request(blob_request)
|
||||
self._blob_download_request = blob_request
|
||||
blob_request.finished_deferred.addCallbacks(self._downloading_finished,
|
||||
self._handle_response_error)
|
||||
return d
|
||||
else:
|
||||
return defer.fail(ValueError("There is already a blob download request active"))
|
||||
|
||||
def cancel_requests(self):
|
||||
self.connection_closing = True
|
||||
ds = []
|
||||
err = RequestCanceledError()
|
||||
for key, d in list(self._response_deferreds.items()):
|
||||
del self._response_deferreds[key]
|
||||
d.errback(err)
|
||||
ds.append(d)
|
||||
if self._blob_download_request is not None:
|
||||
ds.append(self._blob_download_request.finished_deferred)
|
||||
self._blob_download_request.cancel(err)
|
||||
self._blob_download_request = None
|
||||
self._downloading_blob = False
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
######### Internal request handling #########
|
||||
|
||||
def _handle_request_error(self, err):
|
||||
log.error("An unexpected error occurred creating or sending a request to %s. %s: %s",
|
||||
self.peer, err.type, err)
|
||||
self.transport.loseConnection()
|
||||
|
||||
def _ask_for_request(self):
|
||||
if self.connection_closed is True or self.connection_closing is True:
|
||||
return
|
||||
|
||||
def send_request_or_close(do_request):
|
||||
if do_request is True:
|
||||
request_msg, self._next_request = self._next_request, {}
|
||||
self._send_request_message(request_msg)
|
||||
else:
|
||||
# The connection manager has indicated that this connection should be terminated
|
||||
log.debug("Closing the connection to %s due to having no further requests to send",
|
||||
self.peer)
|
||||
self.peer.report_success()
|
||||
self.transport.loseConnection()
|
||||
d = self._connection_manager.get_next_request(self.peer, self)
|
||||
d.addCallback(send_request_or_close)
|
||||
d.addErrback(self._handle_request_error)
|
||||
|
||||
def _send_request_message(self, request_msg):
|
||||
self.setTimeout(self.PROTOCOL_TIMEOUT)
|
||||
# TODO: compare this message to the last one. If they're the same,
|
||||
# TODO: incrementally delay this message.
|
||||
m = json.dumps(request_msg, default=encode_decimal).encode()
|
||||
self.transport.write(m)
|
||||
|
||||
def _get_valid_response(self, response_msg):
|
||||
extra_data = None
|
||||
response = None
|
||||
curr_pos = 0
|
||||
while 1:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren != -1:
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
extra_data = response_msg[curr_pos:]
|
||||
break
|
||||
else:
|
||||
break
|
||||
return response, extra_data
|
||||
|
||||
def _handle_response_error(self, err):
|
||||
# If an error gets to this point, log it and kill the connection.
|
||||
if err.check(DownloadCanceledError, RequestCanceledError, error.ConnectionAborted,
|
||||
ConnectionClosedBeforeResponseError):
|
||||
# TODO: (wish-list) it seems silly to close the connection over this, and it shouldn't
|
||||
# TODO: always be this way. it's done this way now because the client has no other way
|
||||
# TODO: of telling the server it wants the download to stop. It would be great if the
|
||||
# TODO: protocol had such a mechanism.
|
||||
log.info("Closing the connection to %s because the download of blob %s was canceled",
|
||||
self.peer, self._blob_download_request.blob)
|
||||
result = None
|
||||
elif err.check(MisbehavingPeerError):
|
||||
log.warning("The connection to %s is closing due to: %s", self.peer, err)
|
||||
result = err
|
||||
else:
|
||||
log.error("The connection to %s is closing due to an unexpected error: %s",
|
||||
self.peer, err)
|
||||
result = err
|
||||
self._blob_download_request = None
|
||||
self._downloading_blob = False
|
||||
self.transport.loseConnection()
|
||||
return result
|
||||
|
||||
def _handle_response(self, response):
|
||||
ds = []
|
||||
log.debug(
|
||||
"Handling a response from %s. Expected responses: %s. Actual responses: %s",
|
||||
self.peer, self._response_deferreds.keys(), response.keys())
|
||||
for key, val in response.items():
|
||||
if key in self._response_deferreds:
|
||||
d = self._response_deferreds.pop(key)
|
||||
d.callback({key: val})
|
||||
ds.append(d)
|
||||
for k, d in self._response_deferreds.items():
|
||||
del self._response_deferreds[k]
|
||||
d.errback(failure.Failure(NoResponseError()))
|
||||
ds.append(d)
|
||||
|
||||
if self._blob_download_request is not None:
|
||||
self._downloading_blob = True
|
||||
d = self._blob_download_request.finished_deferred
|
||||
d.addErrback(self._handle_response_error)
|
||||
ds.append(d)
|
||||
|
||||
# TODO: are we sure we want to consume errors here
|
||||
dl = defer.DeferredList(ds, consumeErrors=True)
|
||||
|
||||
def get_next_request(results):
|
||||
failed = False
|
||||
for success, result in results:
|
||||
if success is False:
|
||||
failed = True
|
||||
if not isinstance(result.value, DownloadCanceledError):
|
||||
log.info(result.value)
|
||||
log.info("The connection to %s is closing due to an error: %s",
|
||||
self.peer, result.getTraceback())
|
||||
|
||||
self.peer.report_down()
|
||||
if failed is False:
|
||||
log.debug("Asking for another request from %s", self.peer)
|
||||
self._ask_for_request()
|
||||
else:
|
||||
log.warning("Not asking for another request from %s", self.peer)
|
||||
self.transport.loseConnection()
|
||||
|
||||
dl.addCallback(get_next_request)
|
||||
|
||||
def _downloading_finished(self, arg):
|
||||
log.debug("The blob has finished downloading from %s", self.peer)
|
||||
self._blob_download_request = None
|
||||
self._downloading_blob = False
|
||||
return arg
|
||||
|
||||
######### IRateLimited #########
|
||||
|
||||
def throttle_upload(self):
|
||||
pass
|
||||
|
||||
def unthrottle_upload(self):
|
||||
pass
|
||||
|
||||
def throttle_download(self):
|
||||
self.transport.pauseProducing()
|
||||
|
||||
def unthrottle_download(self):
|
||||
self.transport.resumeProducing()
|
||||
|
||||
|
||||
class ClientProtocolFactory(ClientFactory):
|
||||
protocol = ClientProtocol
|
||||
|
||||
def __init__(self, peer, rate_limiter, connection_manager):
|
||||
self.peer = peer
|
||||
self.rate_limiter = rate_limiter
|
||||
self.connection_manager = connection_manager
|
||||
self.p = None
|
||||
# This defer fires and returns True when connection was
|
||||
# made and completed, or fires and returns False if
|
||||
# connection failed
|
||||
self.connection_was_made_deferred = defer.Deferred()
|
||||
|
||||
def clientConnectionFailed(self, connector, reason):
|
||||
log.debug("Connection failed to %s: %s", self.peer, reason)
|
||||
self.peer.report_down()
|
||||
self.connection_was_made_deferred.callback(False)
|
||||
|
||||
def buildProtocol(self, addr):
|
||||
p = self.protocol()
|
||||
p.factory = self
|
||||
self.p = p
|
||||
return p
|
||||
@@ -1,27 +0,0 @@
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
|
||||
|
||||
class ClientRequest:
|
||||
def __init__(self, request_dict, response_identifier=None):
|
||||
self.request_dict = request_dict
|
||||
self.response_identifier = response_identifier
|
||||
|
||||
|
||||
class ClientPaidRequest(ClientRequest):
|
||||
def __init__(self, request_dict, response_identifier, max_pay_units):
|
||||
super().__init__(request_dict, response_identifier)
|
||||
self.max_pay_units = max_pay_units
|
||||
|
||||
|
||||
class ClientBlobRequest(ClientPaidRequest):
|
||||
def __init__(self, request_dict, response_identifier, write_func, finished_deferred,
|
||||
cancel_func, blob):
|
||||
if blob.length is None:
|
||||
max_pay_units = MAX_BLOB_SIZE
|
||||
else:
|
||||
max_pay_units = blob.length
|
||||
super().__init__(request_dict, response_identifier, max_pay_units)
|
||||
self.write = write_func
|
||||
self.finished_deferred = finished_deferred
|
||||
self.cancel = cancel_func
|
||||
self.blob = blob
|
||||
@@ -1,227 +0,0 @@
|
||||
import random
|
||||
import logging
|
||||
from twisted.internet import defer, reactor
|
||||
from lbrynet import conf
|
||||
from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
|
||||
from lbrynet.core.Error import InsufficientFundsError
|
||||
from lbrynet.core import utils
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PeerConnectionHandler:
|
||||
def __init__(self, request_creators, factory):
|
||||
self.request_creators = request_creators
|
||||
self.factory = factory
|
||||
self.connection = None
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
#implements(interfaces.IConnectionManager)
|
||||
MANAGE_CALL_INTERVAL_SEC = 5
|
||||
TCP_CONNECT_TIMEOUT = 15
|
||||
|
||||
def __init__(self, downloader, rate_limiter,
|
||||
primary_request_creators, secondary_request_creators):
|
||||
|
||||
self.seek_head_blob_first = conf.settings['seek_head_blob_first']
|
||||
self.max_connections_per_stream = conf.settings['max_connections_per_stream']
|
||||
|
||||
self.downloader = downloader
|
||||
self.rate_limiter = rate_limiter
|
||||
self._primary_request_creators = primary_request_creators
|
||||
self._secondary_request_creators = secondary_request_creators
|
||||
self._peer_connections = {} # {Peer: PeerConnectionHandler}
|
||||
self._connections_closing = {} # {Peer: deferred (fired when the connection is closed)}
|
||||
self._next_manage_call = None
|
||||
# a deferred that gets fired when a _manage call is set
|
||||
self._manage_deferred = None
|
||||
self.stopped = True
|
||||
log.debug("%s initialized", self._get_log_name())
|
||||
|
||||
# this identifies what the connection manager is for,
|
||||
# used for logging purposes only
|
||||
def _get_log_name(self):
|
||||
out = 'Connection Manager Unknown'
|
||||
if hasattr(self.downloader, 'stream_name'):
|
||||
out = 'Connection Manager '+self.downloader.stream_name
|
||||
elif hasattr(self.downloader, 'blob_hash'):
|
||||
out = 'Connection Manager '+self.downloader.blob_hash
|
||||
return out
|
||||
|
||||
def _start(self):
|
||||
self.stopped = False
|
||||
if self._next_manage_call is not None and self._next_manage_call.active() is True:
|
||||
self._next_manage_call.cancel()
|
||||
|
||||
def start(self):
|
||||
log.debug("%s starting", self._get_log_name())
|
||||
self._start()
|
||||
self._next_manage_call = utils.call_later(0, self.manage)
|
||||
return defer.succeed(True)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
log.debug("%s stopping", self._get_log_name())
|
||||
self.stopped = True
|
||||
# wait for the current manage call to finish
|
||||
if self._manage_deferred:
|
||||
yield self._manage_deferred
|
||||
# in case we stopped between manage calls, cancel the next one
|
||||
if self._next_manage_call and self._next_manage_call.active():
|
||||
self._next_manage_call.cancel()
|
||||
self._next_manage_call = None
|
||||
yield self._close_peers()
|
||||
|
||||
def num_peer_connections(self):
|
||||
return len(self._peer_connections)
|
||||
|
||||
def _close_peers(self):
|
||||
def disconnect_peer(p):
|
||||
d = defer.Deferred()
|
||||
self._connections_closing[p] = d
|
||||
self._peer_connections[p].connection.disconnect()
|
||||
if p in self._peer_connections:
|
||||
del self._peer_connections[p]
|
||||
return d
|
||||
|
||||
def close_connection(p):
|
||||
log.debug("%s Abruptly closing a connection to %s due to downloading being paused",
|
||||
self._get_log_name(), p)
|
||||
if self._peer_connections[p].factory.p is not None:
|
||||
d = self._peer_connections[p].factory.p.cancel_requests()
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addBoth(lambda _: disconnect_peer(p))
|
||||
return d
|
||||
|
||||
# fixme: stop modifying dict during iteration
|
||||
closing_deferreds = [close_connection(peer) for peer in list(self._peer_connections)]
|
||||
return defer.DeferredList(closing_deferreds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_next_request(self, peer, protocol):
|
||||
log.debug("%s Trying to get the next request for peer %s", self._get_log_name(), peer)
|
||||
if not peer in self._peer_connections or self.stopped is True:
|
||||
log.debug("%s The peer %s has already been told to shut down.",
|
||||
self._get_log_name(), peer)
|
||||
defer.returnValue(False)
|
||||
requests = yield self._send_primary_requests(peer, protocol)
|
||||
have_request = any(r[1] for r in requests if r[0] is True)
|
||||
if have_request:
|
||||
yield self._send_secondary_requests(peer, protocol)
|
||||
defer.returnValue(have_request)
|
||||
|
||||
def _send_primary_requests(self, peer, protocol):
|
||||
def handle_error(err):
|
||||
err.trap(InsufficientFundsError)
|
||||
self.downloader.insufficient_funds(err)
|
||||
return False
|
||||
|
||||
def check_if_request_sent(request_sent, request_creator):
|
||||
if peer not in self._peer_connections:
|
||||
# This can happen if the connection is told to close
|
||||
return False
|
||||
if request_sent is False:
|
||||
if request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.remove(request_creator)
|
||||
else:
|
||||
if not request_creator in self._peer_connections[peer].request_creators:
|
||||
self._peer_connections[peer].request_creators.append(request_creator)
|
||||
return request_sent
|
||||
|
||||
ds = []
|
||||
for p_r_c in self._primary_request_creators:
|
||||
d = p_r_c.send_next_request(peer, protocol)
|
||||
d.addErrback(handle_error)
|
||||
d.addCallback(check_if_request_sent, p_r_c)
|
||||
ds.append(d)
|
||||
return defer.DeferredList(ds, fireOnOneErrback=True)
|
||||
|
||||
def _send_secondary_requests(self, peer, protocol):
|
||||
ds = [
|
||||
s_r_c.send_next_request(peer, protocol)
|
||||
for s_r_c in self._secondary_request_creators
|
||||
]
|
||||
return defer.DeferredList(ds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def manage(self, schedule_next_call=True):
|
||||
self._manage_deferred = defer.Deferred()
|
||||
if len(self._peer_connections) < self.max_connections_per_stream:
|
||||
log.debug("%s have %d connections, looking for %d",
|
||||
self._get_log_name(), len(self._peer_connections),
|
||||
self.max_connections_per_stream)
|
||||
peers = yield self._get_new_peers()
|
||||
for peer in peers:
|
||||
self._connect_to_peer(peer)
|
||||
self._manage_deferred.callback(None)
|
||||
self._manage_deferred = None
|
||||
if not self.stopped and schedule_next_call:
|
||||
self._next_manage_call = utils.call_later(self.MANAGE_CALL_INTERVAL_SEC, self.manage)
|
||||
|
||||
def return_shuffled_peers_not_connected_to(self, peers, new_conns_needed):
|
||||
out = [peer for peer in peers if peer not in self._peer_connections]
|
||||
random.shuffle(out)
|
||||
return out[0:new_conns_needed]
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _get_new_peers(self):
|
||||
new_conns_needed = self.max_connections_per_stream - len(self._peer_connections)
|
||||
if new_conns_needed < 1:
|
||||
defer.returnValue([])
|
||||
# we always get the peer from the first request creator
|
||||
# must be a type BlobRequester...
|
||||
request_creator = self._primary_request_creators[0]
|
||||
log.debug("%s Trying to get a new peer to connect to", self._get_log_name())
|
||||
|
||||
# find peers for the head blob if configured to do so
|
||||
if self.seek_head_blob_first:
|
||||
try:
|
||||
peers = yield request_creator.get_new_peers_for_head_blob()
|
||||
peers = self.return_shuffled_peers_not_connected_to(peers, new_conns_needed)
|
||||
except KeyError:
|
||||
log.warning("%s does not have a head blob", self._get_log_name())
|
||||
peers = []
|
||||
else:
|
||||
peers = []
|
||||
|
||||
# we didn't find any new peers on the head blob,
|
||||
# we have to look for the first unavailable blob
|
||||
if not peers:
|
||||
peers = yield request_creator.get_new_peers_for_next_unavailable()
|
||||
peers = self.return_shuffled_peers_not_connected_to(peers, new_conns_needed)
|
||||
|
||||
log.debug("%s Got a list of peers to choose from: %s",
|
||||
self._get_log_name(), peers)
|
||||
log.debug("%s Current connections: %s",
|
||||
self._get_log_name(), self._peer_connections.keys())
|
||||
log.debug("%s List of connection states: %s", self._get_log_name(),
|
||||
[p_c_h.connection.state for p_c_h in self._peer_connections.values()])
|
||||
defer.returnValue(peers)
|
||||
|
||||
def _connect_to_peer(self, peer):
|
||||
if self.stopped:
|
||||
return
|
||||
|
||||
log.debug("%s Trying to connect to %s", self._get_log_name(), peer)
|
||||
factory = ClientProtocolFactory(peer, self.rate_limiter, self)
|
||||
factory.connection_was_made_deferred.addCallback(
|
||||
lambda c_was_made: self._peer_disconnected(c_was_made, peer))
|
||||
self._peer_connections[peer] = PeerConnectionHandler(self._primary_request_creators[:],
|
||||
factory)
|
||||
connection = reactor.connectTCP(peer.host, peer.port, factory,
|
||||
timeout=self.TCP_CONNECT_TIMEOUT)
|
||||
self._peer_connections[peer].connection = connection
|
||||
|
||||
def _peer_disconnected(self, connection_was_made, peer):
|
||||
log.debug("%s protocol disconnected for %s",
|
||||
self._get_log_name(), peer)
|
||||
if peer in self._peer_connections:
|
||||
del self._peer_connections[peer]
|
||||
if peer in self._connections_closing:
|
||||
d = self._connections_closing[peer]
|
||||
del self._connections_closing[peer]
|
||||
d.callback(True)
|
||||
return connection_was_made
|
||||
@@ -1,101 +0,0 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DownloadManager:
|
||||
#implements(interfaces.IDownloadManager)
|
||||
|
||||
def __init__(self, blob_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.blob_info_finder = None
|
||||
self.progress_manager = None
|
||||
self.blob_handler = None
|
||||
self.connection_manager = None
|
||||
self.blobs = {}
|
||||
self.blob_infos = {}
|
||||
|
||||
######### IDownloadManager #########
|
||||
|
||||
def start_downloading(self):
|
||||
d = self.blob_info_finder.get_initial_blobs()
|
||||
log.debug("Requested the initial blobs from the info finder")
|
||||
d.addCallback(self.add_blobs_to_download)
|
||||
d.addCallback(lambda _: self.resume_downloading())
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def resume_downloading(self):
|
||||
yield self.connection_manager.start()
|
||||
yield self.progress_manager.start()
|
||||
defer.returnValue(True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop_downloading(self):
|
||||
yield self.progress_manager.stop()
|
||||
yield self.connection_manager.stop()
|
||||
defer.returnValue(True)
|
||||
|
||||
def add_blobs_to_download(self, blob_infos):
|
||||
|
||||
log.debug("Adding %s blobs to blobs", len(blob_infos))
|
||||
|
||||
def add_blob_to_list(blob, blob_num):
|
||||
self.blobs[blob_num] = blob
|
||||
log.debug(
|
||||
"Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_num)
|
||||
|
||||
def error_during_add(err):
|
||||
log.warning(
|
||||
"An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
ds = []
|
||||
for blob_info in blob_infos:
|
||||
if not blob_info.blob_num in self.blobs:
|
||||
self.blob_infos[blob_info.blob_num] = blob_info
|
||||
log.debug(
|
||||
"Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
d.addCallback(add_blob_to_list, blob_info.blob_num)
|
||||
d.addErrback(error_during_add)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
return dl
|
||||
|
||||
def stream_position(self):
|
||||
return self.progress_manager.stream_position()
|
||||
|
||||
def needed_blobs(self):
|
||||
return self.progress_manager.needed_blobs()
|
||||
|
||||
def final_blob_num(self):
|
||||
return self.blob_info_finder.final_blob_num()
|
||||
|
||||
def handle_blob(self, blob_num):
|
||||
return self.blob_handler.handle_blob(self.blobs[blob_num], self.blob_infos[blob_num])
|
||||
|
||||
def calculate_total_bytes(self):
|
||||
return sum([bi.length for bi in self.blob_infos.values()])
|
||||
|
||||
def calculate_bytes_left_to_output(self):
|
||||
if not self.blobs:
|
||||
return self.calculate_total_bytes()
|
||||
else:
|
||||
to_be_outputted = [
|
||||
b for n, b in self.blobs.items()
|
||||
if n >= self.progress_manager.last_blob_outputted
|
||||
]
|
||||
return sum([b.length for b in to_be_outputted if b.length is not None])
|
||||
|
||||
def calculate_bytes_left_to_download(self):
|
||||
if not self.blobs:
|
||||
return self.calculate_total_bytes()
|
||||
else:
|
||||
return sum([b.length for b in self.needed_blobs() if b.length is not None])
|
||||
|
||||
def get_head_blob_hash(self):
|
||||
return self.blobs[0].blob_hash
|
||||
@@ -1,142 +0,0 @@
|
||||
import logging
|
||||
from lbrynet.core.BlobInfo import BlobInfo
|
||||
from lbrynet.core.client.BlobRequester import BlobRequester
|
||||
from lbrynet.core.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.core.client.DownloadManager import DownloadManager
|
||||
from lbrynet.core.Error import InvalidBlobHashError, DownloadSDTimeout
|
||||
from lbrynet.core.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SingleBlobMetadataHandler:
|
||||
#implements(interfaces.IMetadataHandler)
|
||||
|
||||
def __init__(self, blob_hash, download_manager):
|
||||
self.blob_hash = blob_hash
|
||||
self.download_manager = download_manager
|
||||
|
||||
######## IMetadataHandler #########
|
||||
|
||||
def get_initial_blobs(self):
|
||||
log.debug("Returning the blob info")
|
||||
return defer.succeed([BlobInfo(self.blob_hash, 0, None)])
|
||||
|
||||
def final_blob_num(self):
|
||||
return 0
|
||||
|
||||
|
||||
class SingleProgressManager:
|
||||
def __init__(self, download_manager, finished_callback, timeout_callback, timeout):
|
||||
self.finished_callback = finished_callback
|
||||
self.timeout_callback = timeout_callback
|
||||
self.download_manager = download_manager
|
||||
|
||||
self.timeout = timeout
|
||||
self.timeout_counter = 0
|
||||
self.checker = LoopingCall(self._check_if_finished)
|
||||
|
||||
def start(self):
|
||||
safe_start_looping_call(self.checker, 1)
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
safe_stop_looping_call(self.checker)
|
||||
return defer.succeed(True)
|
||||
|
||||
def _check_if_finished(self):
|
||||
if self.stream_position() == 1:
|
||||
blob_downloaded = self.download_manager.blobs[0]
|
||||
log.debug("The blob %s has been downloaded. Calling the finished callback",
|
||||
str(blob_downloaded))
|
||||
safe_stop_looping_call(self.checker)
|
||||
self.finished_callback(blob_downloaded)
|
||||
elif self.timeout is not None:
|
||||
self.timeout_counter += 1
|
||||
if self.timeout_counter >= self.timeout:
|
||||
safe_stop_looping_call(self.checker)
|
||||
self.timeout_callback()
|
||||
|
||||
def stream_position(self):
|
||||
blobs = self.download_manager.blobs
|
||||
if blobs and blobs[0].get_is_verified():
|
||||
return 1
|
||||
return 0
|
||||
|
||||
def needed_blobs(self):
|
||||
blobs = self.download_manager.blobs
|
||||
assert len(blobs) == 1
|
||||
return [b for b in blobs.values() if not b.get_is_verified()]
|
||||
|
||||
|
||||
class DummyBlobHandler:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def handle_blob(self, blob, blob_info):
|
||||
pass
|
||||
|
||||
|
||||
class StandaloneBlobDownloader:
|
||||
def __init__(self, blob_hash, blob_manager, peer_finder,
|
||||
rate_limiter, payment_rate_manager, wallet,
|
||||
timeout=None):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.timeout = timeout
|
||||
self.download_manager = None
|
||||
self.finished_deferred = None
|
||||
|
||||
def download(self):
|
||||
if not is_valid_blobhash(self.blob_hash):
|
||||
return defer.fail(Failure(InvalidBlobHashError(self.blob_hash)))
|
||||
|
||||
def cancel_download(d):
|
||||
self.stop()
|
||||
|
||||
self.finished_deferred = defer.Deferred(canceller=cancel_download)
|
||||
self.download_manager = DownloadManager(self.blob_manager)
|
||||
self.download_manager.blob_requester = BlobRequester(self.blob_manager, self.peer_finder,
|
||||
self.payment_rate_manager, self.wallet,
|
||||
self.download_manager)
|
||||
self.download_manager.blob_info_finder = SingleBlobMetadataHandler(self.blob_hash,
|
||||
self.download_manager)
|
||||
self.download_manager.progress_manager = SingleProgressManager(self.download_manager,
|
||||
self._blob_downloaded,
|
||||
self._download_timedout,
|
||||
self.timeout)
|
||||
self.download_manager.blob_handler = DummyBlobHandler()
|
||||
self.download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger()
|
||||
self.download_manager.connection_manager = ConnectionManager(
|
||||
self, self.rate_limiter,
|
||||
[self.download_manager.blob_requester],
|
||||
[self.download_manager.wallet_info_exchanger]
|
||||
)
|
||||
d = self.download_manager.start_downloading()
|
||||
d.addCallback(lambda _: self.finished_deferred)
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
return self.download_manager.stop_downloading()
|
||||
|
||||
def _blob_downloaded(self, blob):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.callback(blob)
|
||||
|
||||
def _download_timedout(self):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.errback(DownloadSDTimeout(self.blob_hash))
|
||||
|
||||
def insufficient_funds(self, err):
|
||||
self.stop()
|
||||
if not self.finished_deferred.called:
|
||||
self.finished_deferred.errback(err)
|
||||
@@ -1,165 +0,0 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamProgressManager:
|
||||
#implements(IProgressManager)
|
||||
|
||||
def __init__(self, finished_callback, blob_manager,
|
||||
download_manager, delete_blob_after_finished=False):
|
||||
self.finished_callback = finished_callback
|
||||
self.blob_manager = blob_manager
|
||||
self.delete_blob_after_finished = delete_blob_after_finished
|
||||
self.download_manager = download_manager
|
||||
self.provided_blob_nums = []
|
||||
self.last_blob_outputted = -1
|
||||
self.stopped = True
|
||||
self._next_try_to_output_call = None
|
||||
self.outputting_d = None
|
||||
|
||||
######### IProgressManager #########
|
||||
|
||||
def start(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.stopped = False
|
||||
self._next_try_to_output_call = reactor.callLater(0, self._try_to_output)
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
self.stopped = True
|
||||
if self._next_try_to_output_call is not None and self._next_try_to_output_call.active():
|
||||
self._next_try_to_output_call.cancel()
|
||||
self._next_try_to_output_call = None
|
||||
return self._stop_outputting()
|
||||
|
||||
def blob_downloaded(self, blob, blob_num):
|
||||
if self.outputting_d is None:
|
||||
self._output_loop()
|
||||
|
||||
######### internal #########
|
||||
|
||||
def _finished_outputting(self):
|
||||
self.finished_callback(True)
|
||||
|
||||
def _try_to_output(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self._next_try_to_output_call = reactor.callLater(1, self._try_to_output)
|
||||
if self.outputting_d is None:
|
||||
self._output_loop()
|
||||
|
||||
def _output_loop(self):
|
||||
pass
|
||||
|
||||
def _stop_outputting(self):
|
||||
if self.outputting_d is not None:
|
||||
return self.outputting_d
|
||||
return defer.succeed(None)
|
||||
|
||||
def _finished_with_blob(self, blob_num):
|
||||
log.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
|
||||
if self.delete_blob_after_finished is True:
|
||||
log.debug("delete_blob_after_finished is True")
|
||||
blobs = self.download_manager.blobs
|
||||
if blob_num in blobs:
|
||||
log.debug("Telling the blob manager, %s, to delete blob %s",
|
||||
self.blob_manager, blobs[blob_num].blob_hash)
|
||||
self.blob_manager.delete_blobs([blobs[blob_num].blob_hash])
|
||||
else:
|
||||
log.debug("Blob number %s was not in blobs", str(blob_num))
|
||||
else:
|
||||
log.debug("delete_blob_after_finished is False")
|
||||
|
||||
|
||||
class FullStreamProgressManager(StreamProgressManager):
|
||||
def __init__(self, finished_callback, blob_manager,
|
||||
download_manager, delete_blob_after_finished=False):
|
||||
super().__init__(finished_callback, blob_manager, download_manager,
|
||||
delete_blob_after_finished)
|
||||
self.outputting_d = None
|
||||
|
||||
######### IProgressManager #########
|
||||
|
||||
def _done(self, i, blobs):
|
||||
"""Return true if `i` is a blob number we don't have"""
|
||||
return (
|
||||
i not in blobs or
|
||||
(
|
||||
not blobs[i].get_is_verified() and
|
||||
i not in self.provided_blob_nums
|
||||
)
|
||||
)
|
||||
|
||||
def stream_position(self):
|
||||
blobs = self.download_manager.blobs
|
||||
if not blobs:
|
||||
return 0
|
||||
else:
|
||||
for i in range(max(blobs.keys())):
|
||||
if self._done(i, blobs):
|
||||
return i
|
||||
return max(blobs.keys()) + 1
|
||||
|
||||
def needed_blobs(self):
|
||||
blobs = self.download_manager.blobs
|
||||
return [
|
||||
b for n, b in blobs.items()
|
||||
if not b.get_is_verified() and not n in self.provided_blob_nums
|
||||
]
|
||||
|
||||
######### internal #########
|
||||
|
||||
def _output_loop(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.stopped:
|
||||
if self.outputting_d is not None:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
return
|
||||
|
||||
if self.outputting_d is None:
|
||||
self.outputting_d = defer.Deferred()
|
||||
blobs = self.download_manager.blobs
|
||||
|
||||
def finished_outputting_blob():
|
||||
self.last_blob_outputted += 1
|
||||
|
||||
def check_if_finished():
|
||||
final_blob_num = self.download_manager.final_blob_num()
|
||||
if final_blob_num is not None and final_blob_num == self.last_blob_outputted:
|
||||
self._finished_outputting()
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
else:
|
||||
reactor.callLater(0, self._output_loop)
|
||||
|
||||
current_blob_num = self.last_blob_outputted + 1
|
||||
|
||||
if current_blob_num in blobs and blobs[current_blob_num].get_is_verified():
|
||||
log.debug("Outputting blob %s", str(self.last_blob_outputted + 1))
|
||||
self.provided_blob_nums.append(self.last_blob_outputted + 1)
|
||||
d = self.download_manager.handle_blob(self.last_blob_outputted + 1)
|
||||
d.addCallback(lambda _: finished_outputting_blob())
|
||||
d.addCallback(lambda _: self._finished_with_blob(current_blob_num))
|
||||
d.addCallback(lambda _: check_if_finished())
|
||||
|
||||
def log_error(err):
|
||||
log.warning("Error outputting blob %s: %s", blobs[current_blob_num].blob_hash,
|
||||
err.getErrorMessage())
|
||||
if self.outputting_d is not None and not self.outputting_d.called:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
self.stop()
|
||||
|
||||
d.addErrback(log_error)
|
||||
else:
|
||||
self.outputting_d.callback(True)
|
||||
self.outputting_d = None
|
||||
@@ -1,5 +0,0 @@
|
||||
import hashlib
|
||||
|
||||
|
||||
def get_lbry_hash_obj():
|
||||
return hashlib.sha384()
|
||||
@@ -1,273 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer
|
||||
import twisted.python.log
|
||||
|
||||
from lbrynet import __version__ as lbrynet_version, build_type, conf
|
||||
from lbrynet.core import utils
|
||||
|
||||
|
||||
class HTTPSHandler(logging.Handler):
|
||||
def __init__(self, url, fqdn=False, localname=None, facility=None, cookies=None):
|
||||
super().__init__()
|
||||
self.url = url
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.cookies = cookies or {}
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _emit(self, record):
|
||||
payload = self.format(record)
|
||||
response = yield treq.post(self.url, data=payload, cookies=self.cookies)
|
||||
self.cookies.update(response.cookies())
|
||||
|
||||
def emit(self, record):
|
||||
return self._emit(record)
|
||||
|
||||
|
||||
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
|
||||
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
|
||||
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
|
||||
|
||||
|
||||
def remove_handlers(log, handler_name):
|
||||
for handler in log.handlers:
|
||||
if handler.name == handler_name:
|
||||
log.removeHandler(handler)
|
||||
|
||||
|
||||
def _log_decorator(fn):
|
||||
"""Configure a logging handler.
|
||||
|
||||
`fn` is a function that returns a logging handler. The returned
|
||||
handler has its log-level set and is attached to the specified
|
||||
logger or the root logger.
|
||||
"""
|
||||
|
||||
def helper(*args, **kwargs):
|
||||
log = kwargs.pop('log', logging.getLogger())
|
||||
level = kwargs.pop('level', logging.INFO)
|
||||
if not isinstance(level, int):
|
||||
# despite the name, getLevelName returns
|
||||
# the numeric level when passed a text level
|
||||
level = logging.getLevelName(level)
|
||||
handler = fn(*args, **kwargs)
|
||||
configure_handler(handler, log, level)
|
||||
return handler
|
||||
|
||||
return helper
|
||||
|
||||
|
||||
def configure_handler(handler, log, level):
|
||||
if handler.name:
|
||||
remove_handlers(log, handler.name)
|
||||
handler.setLevel(level)
|
||||
log.addHandler(handler)
|
||||
# need to reduce the logger's level down to the
|
||||
# handler's level or else the handler won't
|
||||
# get those messages
|
||||
if log.level > level:
|
||||
log.setLevel(level)
|
||||
return handler
|
||||
|
||||
|
||||
def disable_third_party_loggers():
|
||||
logging.getLogger('requests').setLevel(logging.CRITICAL)
|
||||
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
|
||||
logging.getLogger('BitcoinRPC').setLevel(logging.INFO)
|
||||
logging.getLogger('lbryum').setLevel(logging.WARNING)
|
||||
logging.getLogger('twisted').setLevel(logging.CRITICAL)
|
||||
logging.getLogger('aioupnp').setLevel(logging.WARNING)
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_console(**kwargs):
|
||||
"""Convenience function to configure a log-handler that outputs to stdout"""
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'console'
|
||||
return handler
|
||||
|
||||
|
||||
@_log_decorator
|
||||
def configure_file_handler(file_name, **kwargs):
|
||||
"""Convenience function to configure a log-handler that writes to `file_name`"""
|
||||
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
|
||||
handler.setFormatter(DEFAULT_FORMATTER)
|
||||
handler.name = 'file'
|
||||
return handler
|
||||
|
||||
|
||||
def get_loggly_url(token=None, version=None):
|
||||
token = token or utils.deobfuscate(conf.settings['LOGGLY_TOKEN'])
|
||||
version = version or lbrynet_version
|
||||
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
|
||||
|
||||
|
||||
def configure_loggly_handler():
|
||||
if build_type.BUILD == 'dev':
|
||||
return
|
||||
if not conf.settings['share_usage_data']:
|
||||
return
|
||||
level = logging.ERROR
|
||||
handler = get_loggly_handler(level=level, installation_id=conf.settings.installation_id,
|
||||
session_id=conf.settings.get_session_id())
|
||||
log = logging.getLogger("lbrynet")
|
||||
if handler.name:
|
||||
remove_handlers(log, handler.name)
|
||||
handler.setLevel(level)
|
||||
log.addHandler(handler)
|
||||
# need to reduce the logger's level down to the
|
||||
# handler's level or else the handler won't
|
||||
# get those messages
|
||||
if log.level > level:
|
||||
log.setLevel(level)
|
||||
|
||||
|
||||
def get_loggly_handler(level, installation_id, session_id):
|
||||
formatter = JsonFormatter(level=level, installation_id=installation_id, session_id=session_id)
|
||||
handler = HTTPSHandler(get_loggly_url())
|
||||
handler.setFormatter(formatter)
|
||||
handler.name = 'loggly'
|
||||
return handler
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
def failure(failure, log, msg, *args):
|
||||
"""Log a failure message from a deferred.
|
||||
|
||||
Args:
|
||||
failure: twisted.python.failure.Failure
|
||||
log: a python logger instance
|
||||
msg: the message to log. Can use normal logging string interpolation.
|
||||
the last argument will be set to the error message from the failure.
|
||||
args: values to substitute into `msg`
|
||||
"""
|
||||
args += (failure.getErrorMessage(),)
|
||||
exc_info = (failure.type, failure.value, failure.getTracebackObject())
|
||||
log.error(msg, *args, exc_info=exc_info)
|
||||
|
||||
|
||||
def convert_verbose(verbose):
|
||||
"""Convert the results of the --verbose flag into a list of logger names
|
||||
|
||||
if --verbose is not provided, args.verbose will be None and logging
|
||||
should be at the info level.
|
||||
if --verbose is provided, but not followed by any arguments, then
|
||||
args.verbose = [] and debug logging should be enabled for all of lbrynet
|
||||
along with info logging on lbryum.
|
||||
if --verbose is provided and followed by arguments, those arguments
|
||||
will be in a list
|
||||
"""
|
||||
if verbose is None:
|
||||
return []
|
||||
if verbose == []:
|
||||
return ['lbrynet', 'lbryum']
|
||||
return verbose
|
||||
|
||||
|
||||
def configure_logging(file_name, console, verbose=None):
|
||||
"""Apply the default logging configuration.
|
||||
|
||||
Enables two log-handlers at the INFO level: a file logger and a loggly logger.
|
||||
Optionally turns on a console logger that defaults to the INFO level, with
|
||||
specified loggers being set to the DEBUG level.
|
||||
|
||||
Args:
|
||||
file_name: the file to which logs should be saved
|
||||
console: If true, enable a console logger
|
||||
verbose: a list of loggers to set to debug level.
|
||||
See `convert_verbose` for more details.
|
||||
"""
|
||||
verbose = convert_verbose(verbose)
|
||||
configure_twisted()
|
||||
configure_file_handler(file_name)
|
||||
disable_third_party_loggers()
|
||||
if console:
|
||||
# if there are some loggers at the debug level, we need
|
||||
# to enable the console to allow debug. Otherwise, only
|
||||
# allow info.
|
||||
level = 'DEBUG' if verbose else 'INFO'
|
||||
handler = configure_console(level=level)
|
||||
if 'lbryum' in verbose:
|
||||
# TODO: this enables lbryum logging on the other handlers
|
||||
# too which isn't consistent with how verbose logging
|
||||
# happens with other loggers. Should change the configuration
|
||||
# so that its only logging at the INFO level for the console.
|
||||
logging.getLogger('lbryum').setLevel(logging.INFO)
|
||||
verbose.remove('lbryum')
|
||||
if verbose:
|
||||
handler.addFilter(LoggerNameFilter(verbose))
|
||||
|
||||
|
||||
def configure_twisted():
|
||||
"""Setup twisted logging to output events to the python stdlib logger"""
|
||||
# I tried using the new logging api
|
||||
# https://twistedmatrix.com/documents/current/core/howto/logger.html#compatibility-with-standard-library-logging
|
||||
# and it simply didn't work
|
||||
observer = twisted.python.log.PythonLoggingObserver()
|
||||
observer.start()
|
||||
|
||||
|
||||
class LoggerNameFilter:
|
||||
"""Filter a log record based on its name.
|
||||
|
||||
Allows all info level and higher records to pass through.
|
||||
Debug records pass if the log record name (or a parent) match
|
||||
the input list of logger names.
|
||||
"""
|
||||
|
||||
def __init__(self, logger_names):
|
||||
self.logger_names = logger_names
|
||||
|
||||
def filter(self, record):
|
||||
if record.levelno >= logging.INFO:
|
||||
return True
|
||||
name = record.name
|
||||
while name:
|
||||
if name in self.logger_names:
|
||||
return True
|
||||
name = get_parent(name)
|
||||
return False
|
||||
|
||||
|
||||
def get_parent(logger_name):
|
||||
names = logger_name.split('.')
|
||||
if len(names) == 1:
|
||||
return ''
|
||||
names = names[:-1]
|
||||
return '.'.join(names)
|
||||
@@ -1,20 +0,0 @@
|
||||
class LoopingCallManager:
|
||||
def __init__(self, calls=None):
|
||||
self.calls = calls or {}
|
||||
|
||||
def register_looping_call(self, name, call):
|
||||
assert name not in self.calls, '{} is already registered'.format(name)
|
||||
self.calls[name] = call
|
||||
|
||||
def start(self, name, *args):
|
||||
lcall = self.calls[name]
|
||||
if not lcall.running:
|
||||
lcall.start(*args)
|
||||
|
||||
def stop(self, name):
|
||||
self.calls[name].stop()
|
||||
|
||||
def shutdown(self):
|
||||
for lcall in self.calls.values():
|
||||
if lcall.running:
|
||||
lcall.stop()
|
||||
@@ -1,57 +0,0 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAvailabilityHandlerFactory:
|
||||
# implements(IQueryHandlerFactory)
|
||||
|
||||
def __init__(self, blob_manager):
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
######### IQueryHandlerFactory #########
|
||||
|
||||
def build_query_handler(self):
|
||||
q_h = BlobAvailabilityHandler(self.blob_manager)
|
||||
return q_h
|
||||
|
||||
def get_primary_query_identifier(self):
|
||||
return 'requested_blobs'
|
||||
|
||||
def get_description(self):
|
||||
return "Blob Availability - blobs that are available to be uploaded"
|
||||
|
||||
|
||||
class BlobAvailabilityHandler:
|
||||
#implements(IQueryHandler)
|
||||
|
||||
def __init__(self, blob_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.query_identifiers = ['requested_blobs']
|
||||
|
||||
######### IQueryHandler #########
|
||||
|
||||
def register_with_request_handler(self, request_handler, peer):
|
||||
request_handler.register_query_handler(self, self.query_identifiers)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
log.info("Received the client's list of requested blobs")
|
||||
d = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
|
||||
def set_field(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
|
||||
d.addCallback(set_field)
|
||||
return d
|
||||
return defer.succeed({})
|
||||
|
||||
######### internal #########
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
|
||||
return d
|
||||
@@ -1,227 +0,0 @@
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from lbrynet import analytics
|
||||
from lbrynet.core.Offer import Offer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobRequestHandlerFactory:
|
||||
#implements(IQueryHandlerFactory)
|
||||
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager, analytics_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
######### IQueryHandlerFactory #########
|
||||
|
||||
def build_query_handler(self):
|
||||
q_h = BlobRequestHandler(
|
||||
self.blob_manager, self.wallet, self.payment_rate_manager, self.analytics_manager)
|
||||
return q_h
|
||||
|
||||
def get_primary_query_identifier(self):
|
||||
return 'requested_blob'
|
||||
|
||||
def get_description(self):
|
||||
return "Blob Uploader - uploads blobs"
|
||||
|
||||
|
||||
class BlobRequestHandler:
|
||||
#implements(IQueryHandler, IBlobSender)
|
||||
PAYMENT_RATE_QUERY = 'blob_data_payment_rate'
|
||||
BLOB_QUERY = 'requested_blob'
|
||||
AVAILABILITY_QUERY = 'requested_blobs'
|
||||
|
||||
def __init__(self, blob_manager, wallet, payment_rate_manager, analytics_manager):
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.query_identifiers = [self.PAYMENT_RATE_QUERY, self.BLOB_QUERY, self.AVAILABILITY_QUERY]
|
||||
self.analytics_manager = analytics_manager
|
||||
self.peer = None
|
||||
self.blob_data_payment_rate = None
|
||||
self.read_handle = None
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
self.blob_bytes_uploaded = 0
|
||||
self._blobs_requested = []
|
||||
|
||||
######### IQueryHandler #########
|
||||
|
||||
def register_with_request_handler(self, request_handler, peer):
|
||||
self.peer = peer
|
||||
request_handler.register_query_handler(self, self.query_identifiers)
|
||||
request_handler.register_blob_sender(self)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
response = defer.succeed({})
|
||||
log.debug("Handle query: %s", str(queries))
|
||||
|
||||
if self.AVAILABILITY_QUERY in queries:
|
||||
self._blobs_requested = queries[self.AVAILABILITY_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_availability(r, self._blobs_requested))
|
||||
if self.PAYMENT_RATE_QUERY in queries:
|
||||
offered_rate = queries[self.PAYMENT_RATE_QUERY]
|
||||
offer = Offer(offered_rate)
|
||||
if offer.rate is None:
|
||||
log.warning("Empty rate offer")
|
||||
response.addCallback(lambda r: self._handle_payment_rate_query(offer, r))
|
||||
if self.BLOB_QUERY in queries:
|
||||
incoming = queries[self.BLOB_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_send_request(r, incoming))
|
||||
return response
|
||||
|
||||
######### IBlobSender #########
|
||||
|
||||
def send_blob_if_requested(self, consumer):
|
||||
if self.currently_uploading is not None:
|
||||
return self.send_file(consumer)
|
||||
return defer.succeed(True)
|
||||
|
||||
def cancel_send(self, err):
|
||||
if self.currently_uploading is not None:
|
||||
self.read_handle.close()
|
||||
self.read_handle = None
|
||||
self.currently_uploading = None
|
||||
return err
|
||||
|
||||
######### internal #########
|
||||
|
||||
def _reply_to_availability(self, request, blobs):
|
||||
d = self._get_available_blobs(blobs)
|
||||
|
||||
def set_available(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
request.update({'available_blobs': available_blobs})
|
||||
return request
|
||||
|
||||
d.addCallback(set_available)
|
||||
return d
|
||||
|
||||
def _handle_payment_rate_query(self, offer, request):
|
||||
blobs = self._blobs_requested
|
||||
log.debug("Offered rate %f LBC/mb for %i blobs", offer.rate, len(blobs))
|
||||
reply = self.payment_rate_manager.reply_to_offer(self.peer, blobs, offer)
|
||||
if reply.is_accepted:
|
||||
self.blob_data_payment_rate = offer.rate
|
||||
request[self.PAYMENT_RATE_QUERY] = "RATE_ACCEPTED"
|
||||
log.debug("Accepted rate: %f", offer.rate)
|
||||
elif reply.is_too_low:
|
||||
request[self.PAYMENT_RATE_QUERY] = "RATE_TOO_LOW"
|
||||
log.debug("Reject rate: %f", offer.rate)
|
||||
elif reply.is_unset:
|
||||
log.warning("Rate unset")
|
||||
request['incoming_blob'] = {'error': 'RATE_UNSET'}
|
||||
log.debug("Returning rate query result: %s", str(request))
|
||||
|
||||
return request
|
||||
|
||||
def _handle_blob_query(self, response, query):
|
||||
log.debug("Received the client's request to send a blob")
|
||||
response['incoming_blob'] = {}
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
response['incoming_blob'] = {'error': "RATE_UNSET"}
|
||||
return response
|
||||
else:
|
||||
return self._send_blob(response, query)
|
||||
|
||||
def _send_blob(self, response, query):
|
||||
d = self.blob_manager.get_blob(query)
|
||||
d.addCallback(self.open_blob_for_reading, response)
|
||||
return d
|
||||
|
||||
def open_blob_for_reading(self, blob, response):
|
||||
response_fields = {}
|
||||
d = defer.succeed(None)
|
||||
if blob.get_is_verified():
|
||||
read_handle = blob.open_for_reading()
|
||||
if read_handle is not None:
|
||||
self.currently_uploading = blob
|
||||
self.read_handle = read_handle
|
||||
log.info("Sending %s to %s", str(blob), self.peer)
|
||||
response_fields['blob_hash'] = blob.blob_hash
|
||||
response_fields['length'] = blob.length
|
||||
response['incoming_blob'] = response_fields
|
||||
d.addCallback(lambda _: response)
|
||||
return d
|
||||
log.debug("We can not send %s", str(blob))
|
||||
response['incoming_blob'] = {'error': 'BLOB_UNAVAILABLE'}
|
||||
d.addCallback(lambda _: response)
|
||||
return d
|
||||
|
||||
def _reply_to_send_request(self, response, incoming):
|
||||
response_fields = {}
|
||||
response['incoming_blob'] = response_fields
|
||||
|
||||
if self.blob_data_payment_rate is None:
|
||||
log.debug("Rate not set yet")
|
||||
response['incoming_blob'] = {'error': 'RATE_UNSET'}
|
||||
return defer.succeed(response)
|
||||
else:
|
||||
log.debug("Requested blob: %s", str(incoming))
|
||||
d = self.blob_manager.get_blob(incoming)
|
||||
d.addCallback(lambda blob: self.open_blob_for_reading(blob, response))
|
||||
return d
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
return d
|
||||
|
||||
def send_file(self, consumer):
|
||||
|
||||
def _send_file():
|
||||
inner_d = start_transfer()
|
||||
# TODO: if the transfer fails, check if it's because the connection was cut off.
|
||||
# TODO: if so, perhaps bill the client
|
||||
inner_d.addCallback(lambda _: set_expected_payment())
|
||||
inner_d.addBoth(set_not_uploading)
|
||||
return inner_d
|
||||
|
||||
def count_bytes(data):
|
||||
uploaded = len(data)
|
||||
self.blob_bytes_uploaded += uploaded
|
||||
self.peer.update_stats('blob_bytes_uploaded', uploaded)
|
||||
if self.analytics_manager is not None:
|
||||
self.analytics_manager.add_observation(analytics.BLOB_BYTES_UPLOADED, uploaded)
|
||||
return data
|
||||
|
||||
def start_transfer():
|
||||
self.file_sender = FileSender()
|
||||
log.debug("Starting the file upload")
|
||||
assert self.read_handle is not None, \
|
||||
"self.read_handle was None when trying to start the transfer"
|
||||
d = self.file_sender.beginFileTransfer(self.read_handle, consumer, count_bytes)
|
||||
return d
|
||||
|
||||
def set_expected_payment():
|
||||
log.debug("Setting expected payment")
|
||||
if (
|
||||
self.blob_bytes_uploaded != 0 and self.blob_data_payment_rate is not None
|
||||
and self.blob_data_payment_rate > 0
|
||||
):
|
||||
# TODO: explain why 2**20
|
||||
self.wallet.add_expected_payment(self.peer,
|
||||
self.currently_uploading.length * 1.0 *
|
||||
self.blob_data_payment_rate / 2 ** 20)
|
||||
self.blob_bytes_uploaded = 0
|
||||
self.peer.update_stats('blobs_uploaded', 1)
|
||||
return None
|
||||
|
||||
def set_not_uploading(reason=None):
|
||||
if self.currently_uploading is not None:
|
||||
self.read_handle.close()
|
||||
self.read_handle = None
|
||||
self.currently_uploading = None
|
||||
self.file_sender = None
|
||||
if reason is not None and isinstance(reason, Failure):
|
||||
log.warning("Upload has failed. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
return _send_file()
|
||||
@@ -1,92 +0,0 @@
|
||||
import logging
|
||||
from twisted.internet import error
|
||||
from twisted.internet.protocol import Protocol, ServerFactory
|
||||
from twisted.python import failure
|
||||
from lbrynet.core.server.ServerRequestHandler import ServerRequestHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerProtocol(Protocol):
|
||||
"""ServerProtocol needs to:
|
||||
|
||||
1) Receive requests from its transport
|
||||
2) Pass those requests on to its request handler
|
||||
3) Tell the request handler to pause/resume producing
|
||||
4) Tell its transport to pause/resume producing
|
||||
5) Hang up when the request handler is done producing
|
||||
6) Tell the request handler to stop producing if the connection is lost
|
||||
7) Upon creation, register with the rate limiter
|
||||
8) Upon connection loss, unregister with the rate limiter
|
||||
9) Report all uploaded and downloaded bytes to the rate limiter
|
||||
10) Pause/resume production when told by the rate limiter
|
||||
"""
|
||||
|
||||
#implements(interfaces.IConsumer)
|
||||
|
||||
#Protocol stuff
|
||||
|
||||
def connectionMade(self):
|
||||
log.debug("Got a connection")
|
||||
peer_info = self.transport.getPeer()
|
||||
self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port)
|
||||
self.request_handler = ServerRequestHandler(self)
|
||||
for query_handler_factory in self.factory.query_handler_factories.values():
|
||||
query_handler = query_handler_factory.build_query_handler()
|
||||
query_handler.register_with_request_handler(self.request_handler, self.peer)
|
||||
log.debug("Setting the request handler")
|
||||
self.factory.rate_limiter.register_protocol(self)
|
||||
|
||||
def connectionLost(self, reason=failure.Failure(error.ConnectionDone())):
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.stopProducing()
|
||||
self.factory.rate_limiter.unregister_protocol(self)
|
||||
if not reason.check(error.ConnectionDone):
|
||||
log.warning("Closing a connection. Reason: %s", reason.getErrorMessage())
|
||||
|
||||
def dataReceived(self, data):
|
||||
log.debug("Receiving %s bytes of data from the transport", str(len(data)))
|
||||
self.factory.rate_limiter.report_dl_bytes(len(data))
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.data_received(data)
|
||||
|
||||
#IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
log.debug("Registering the producer")
|
||||
assert streaming is True
|
||||
|
||||
def unregisterProducer(self):
|
||||
self.request_handler = None
|
||||
self.transport.loseConnection()
|
||||
|
||||
def write(self, data):
|
||||
log.trace("Writing %s bytes of data to the transport", len(data))
|
||||
self.transport.write(data)
|
||||
self.factory.rate_limiter.report_ul_bytes(len(data))
|
||||
|
||||
#Rate limiter stuff
|
||||
|
||||
def throttle_upload(self):
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.pauseProducing()
|
||||
|
||||
def unthrottle_upload(self):
|
||||
if self.request_handler is not None:
|
||||
self.request_handler.resumeProducing()
|
||||
|
||||
def throttle_download(self):
|
||||
self.transport.pauseProducing()
|
||||
|
||||
def unthrottle_download(self):
|
||||
self.transport.resumeProducing()
|
||||
|
||||
|
||||
class ServerProtocolFactory(ServerFactory):
|
||||
protocol = ServerProtocol
|
||||
|
||||
def __init__(self, rate_limiter, query_handler_factories, peer_manager):
|
||||
self.rate_limiter = rate_limiter
|
||||
self.query_handler_factories = query_handler_factories
|
||||
self.peer_manager = peer_manager
|
||||
@@ -1,184 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServerRequestHandler:
|
||||
"""This class handles requests from clients. It can upload blobs and
|
||||
return request for information about more blobs that are
|
||||
associated with streams.
|
||||
"""
|
||||
#implements(interfaces.IPushProducer, interfaces.IConsumer, IRequestHandler)
|
||||
|
||||
def __init__(self, consumer):
|
||||
self.consumer = consumer
|
||||
self.production_paused = False
|
||||
self.request_buff = b''
|
||||
self.response_buff = b''
|
||||
self.producer = None
|
||||
self.request_received = False
|
||||
self.CHUNK_SIZE = 2**14
|
||||
self.query_handlers = {} # {IQueryHandler: [query_identifiers]}
|
||||
self.blob_sender = None
|
||||
self.consumer.registerProducer(self, True)
|
||||
|
||||
#IPushProducer stuff
|
||||
|
||||
def pauseProducing(self):
|
||||
self.production_paused = True
|
||||
|
||||
def stopProducing(self):
|
||||
if self.producer is not None:
|
||||
self.producer.stopProducing()
|
||||
self.producer = None
|
||||
self.production_paused = True
|
||||
self.consumer.unregisterProducer()
|
||||
|
||||
def resumeProducing(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.production_paused = False
|
||||
self._produce_more()
|
||||
if self.producer is not None:
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def _produce_more(self):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
if self.production_paused:
|
||||
return
|
||||
chunk = self.response_buff[:self.CHUNK_SIZE]
|
||||
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
|
||||
if chunk == b'':
|
||||
return
|
||||
log.trace("writing %s bytes to the client", len(chunk))
|
||||
self.consumer.write(chunk)
|
||||
reactor.callLater(0, self._produce_more)
|
||||
|
||||
#IConsumer stuff
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
self.producer = producer
|
||||
assert streaming is False
|
||||
producer.resumeProducing()
|
||||
|
||||
def unregisterProducer(self):
|
||||
self.producer = None
|
||||
|
||||
def write(self, data):
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.response_buff = self.response_buff + data
|
||||
self._produce_more()
|
||||
|
||||
def get_more_data():
|
||||
if self.producer is not None:
|
||||
log.trace("Requesting more data from the producer")
|
||||
self.producer.resumeProducing()
|
||||
|
||||
reactor.callLater(0, get_more_data)
|
||||
|
||||
#From Protocol
|
||||
|
||||
def data_received(self, data):
|
||||
log.debug("Received data")
|
||||
log.debug("%s", str(data))
|
||||
if self.request_received is False:
|
||||
return self._parse_data_and_maybe_send_blob(data)
|
||||
else:
|
||||
log.warning(
|
||||
"The client sent data when we were uploading a file. This should not happen")
|
||||
|
||||
def _parse_data_and_maybe_send_blob(self, data):
|
||||
self.request_buff = self.request_buff + data
|
||||
msg = self.try_to_parse_request(self.request_buff)
|
||||
if msg:
|
||||
self.request_buff = b''
|
||||
self._process_msg(msg)
|
||||
else:
|
||||
log.debug("Request buff not a valid json message")
|
||||
log.debug("Request buff: %s", self.request_buff)
|
||||
|
||||
def _process_msg(self, msg):
|
||||
d = self.handle_request(msg)
|
||||
if self.blob_sender:
|
||||
d.addCallback(lambda _: self.blob_sender.send_blob_if_requested(self))
|
||||
d.addCallbacks(lambda _: self.finished_response(), self.request_failure_handler)
|
||||
|
||||
|
||||
######### IRequestHandler #########
|
||||
|
||||
def register_query_handler(self, query_handler, query_identifiers):
|
||||
self.query_handlers[query_handler] = query_identifiers
|
||||
|
||||
def register_blob_sender(self, blob_sender):
|
||||
self.blob_sender = blob_sender
|
||||
|
||||
#response handling
|
||||
|
||||
def request_failure_handler(self, err):
|
||||
log.warning("An error occurred handling a request. Error: %s", err.getErrorMessage())
|
||||
self.stopProducing()
|
||||
return err
|
||||
|
||||
def finished_response(self):
|
||||
self.request_received = False
|
||||
self._produce_more()
|
||||
|
||||
def send_response(self, msg):
|
||||
m = json.dumps(msg).encode()
|
||||
log.debug("Sending a response of length %s", str(len(m)))
|
||||
log.debug("Response: %s", str(m))
|
||||
self.response_buff = self.response_buff + m
|
||||
self._produce_more()
|
||||
return True
|
||||
|
||||
def handle_request(self, msg):
|
||||
log.debug("Handling a request")
|
||||
log.debug(str(msg))
|
||||
|
||||
def create_response_message(results):
|
||||
response = {}
|
||||
for success, result in results:
|
||||
if success is True:
|
||||
response.update(result)
|
||||
else:
|
||||
# result is a Failure
|
||||
return result
|
||||
log.debug("Finished making the response message. Response: %s", str(response))
|
||||
return response
|
||||
|
||||
def log_errors(err):
|
||||
log.warning(
|
||||
"An error occurred handling a client request. Error message: %s",
|
||||
err.getErrorMessage())
|
||||
return err
|
||||
|
||||
def send_response(response):
|
||||
self.send_response(response)
|
||||
return True
|
||||
|
||||
ds = []
|
||||
for query_handler, query_identifiers in self.query_handlers.items():
|
||||
queries = {q_i: msg[q_i] for q_i in query_identifiers if q_i in msg}
|
||||
d = query_handler.handle_queries(queries)
|
||||
d.addErrback(log_errors)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
dl.addCallback(create_response_message)
|
||||
dl.addCallback(send_response)
|
||||
return dl
|
||||
|
||||
def try_to_parse_request(self, request_buff):
|
||||
try:
|
||||
msg = json.loads(request_buff)
|
||||
return msg
|
||||
except ValueError:
|
||||
return None
|
||||
@@ -1,59 +0,0 @@
|
||||
import platform
|
||||
import json
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
from six.moves.urllib import request
|
||||
from six.moves.urllib.error import URLError
|
||||
from lbryschema import __version__ as lbryschema_version
|
||||
from lbrynet import build_type, __version__ as lbrynet_version
|
||||
from lbrynet.conf import ROOT_DIR
|
||||
import logging.handlers
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_lbrynet_version() -> str:
|
||||
if build_type.BUILD == "dev":
|
||||
try:
|
||||
with open(os.devnull, 'w') as devnull:
|
||||
git_dir = ROOT_DIR + '/.git'
|
||||
return subprocess.check_output(
|
||||
['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'],
|
||||
stderr=devnull
|
||||
).decode().strip().lstrip('v')
|
||||
except (subprocess.CalledProcessError, OSError):
|
||||
log.debug("failed to get version from git")
|
||||
return lbrynet_version
|
||||
|
||||
|
||||
def get_platform(get_ip: bool = True) -> dict:
|
||||
p = {
|
||||
"processor": platform.processor(),
|
||||
"python_version": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"os_release": platform.release(),
|
||||
"os_system": platform.system(),
|
||||
"lbrynet_version": get_lbrynet_version(),
|
||||
"lbryschema_version": lbryschema_version,
|
||||
"build": build_type.BUILD, # CI server sets this during build step
|
||||
}
|
||||
if p["os_system"] == "Linux":
|
||||
try:
|
||||
import distro
|
||||
p["distro"] = distro.info()
|
||||
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
|
||||
except ModuleNotFoundError:
|
||||
pass
|
||||
|
||||
# TODO: remove this from get_platform and add a get_external_ip function using treq
|
||||
if get_ip:
|
||||
try:
|
||||
response = json.loads(request.urlopen("https://api.lbry.io/ip").read())
|
||||
if not response['success']:
|
||||
raise URLError("failed to get external ip")
|
||||
p['ip'] = response['data']['ip']
|
||||
except (URLError, AssertionError):
|
||||
p['ip'] = "Could not determine IP"
|
||||
|
||||
return p
|
||||
@@ -1,249 +0,0 @@
|
||||
import base64
|
||||
import codecs
|
||||
import datetime
|
||||
import random
|
||||
import socket
|
||||
import string
|
||||
import json
|
||||
import traceback
|
||||
import functools
|
||||
import logging
|
||||
import pkg_resources
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet import defer
|
||||
from lbryschema.claim import ClaimDict
|
||||
from lbrynet.core.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
|
||||
|
||||
# defining these time functions here allows for easier overriding in testing
|
||||
def now():
|
||||
return datetime.datetime.now()
|
||||
|
||||
|
||||
def utcnow():
|
||||
return datetime.datetime.utcnow()
|
||||
|
||||
|
||||
def isonow():
|
||||
"""Return utc now in isoformat with timezone"""
|
||||
return utcnow().isoformat() + 'Z'
|
||||
|
||||
|
||||
def today():
|
||||
return datetime.datetime.today()
|
||||
|
||||
|
||||
def timedelta(**kwargs):
|
||||
return datetime.timedelta(**kwargs)
|
||||
|
||||
|
||||
def datetime_obj(*args, **kwargs):
|
||||
return datetime.datetime(*args, **kwargs)
|
||||
|
||||
|
||||
def call_later(delay, func, *args, **kwargs):
|
||||
# Import here to ensure that it gets called after installing a reactor
|
||||
# see: http://twistedmatrix.com/documents/current/core/howto/choosing-reactor.html
|
||||
from twisted.internet import reactor
|
||||
return reactor.callLater(delay, func, *args, **kwargs)
|
||||
|
||||
def safe_start_looping_call(looping_call, interval_sec):
|
||||
if not looping_call.running:
|
||||
looping_call.start(interval_sec)
|
||||
|
||||
def safe_stop_looping_call(looping_call):
|
||||
if looping_call.running:
|
||||
looping_call.stop()
|
||||
|
||||
def generate_id(num=None):
|
||||
h = get_lbry_hash_obj()
|
||||
if num is not None:
|
||||
h.update(str(num).encode())
|
||||
else:
|
||||
h.update(str(random.getrandbits(512)).encode())
|
||||
return h.digest()
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char):
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash):
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@param blobhash: string, the blobhash to check
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
def version_is_greater_than(a, b):
|
||||
"""Returns True if version a is more recent than version b"""
|
||||
return pkg_resources.parse_version(a) > pkg_resources.parse_version(b)
|
||||
|
||||
|
||||
def rot13(some_str):
|
||||
return codecs.encode(some_str, 'rot_13')
|
||||
|
||||
|
||||
def deobfuscate(obfustacated):
|
||||
return base64.b64decode(rot13(obfustacated)).decode()
|
||||
|
||||
|
||||
def obfuscate(plain):
|
||||
return rot13(base64.b64encode(plain).decode())
|
||||
|
||||
|
||||
def check_connection(server="lbry.io", port=80, timeout=5):
|
||||
"""Attempts to open a socket to server:port and returns True if successful."""
|
||||
log.debug('Checking connection to %s:%s', server, port)
|
||||
try:
|
||||
server = socket.gethostbyname(server)
|
||||
conn = socket.create_connection((server, port), timeout)
|
||||
conn.close()
|
||||
log.debug('Connection successful')
|
||||
return True
|
||||
except (socket.gaierror, socket.herror) as ex:
|
||||
log.warning("Failed to connect to %s:%s. Unable to resolve domain. Trying to bypass DNS",
|
||||
server, port)
|
||||
try:
|
||||
server = "8.8.8.8"
|
||||
port = 53
|
||||
socket.create_connection((server, port), timeout)
|
||||
log.debug('Connection successful')
|
||||
return True
|
||||
except Exception as ex:
|
||||
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
|
||||
server, port)
|
||||
return False
|
||||
except Exception as ex:
|
||||
log.error("Failed to connect to %s:%s. Maybe the internet connection is not working",
|
||||
server, port)
|
||||
return False
|
||||
|
||||
|
||||
def random_string(length=10, chars=string.ascii_lowercase):
|
||||
return ''.join([random.choice(chars) for _ in range(length)])
|
||||
|
||||
|
||||
def short_hash(hash_str):
|
||||
return hash_str[:6]
|
||||
|
||||
|
||||
def get_sd_hash(stream_info):
|
||||
if not stream_info:
|
||||
return None
|
||||
if isinstance(stream_info, ClaimDict):
|
||||
return stream_info.source_hash
|
||||
result = stream_info.get('claim', {}).\
|
||||
get('value', {}).\
|
||||
get('stream', {}).\
|
||||
get('source', {}).\
|
||||
get('source')
|
||||
if not result:
|
||||
log.warning("Unable to get sd_hash")
|
||||
return result
|
||||
|
||||
|
||||
def json_dumps_pretty(obj, **kwargs):
|
||||
return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs)
|
||||
|
||||
|
||||
class DeferredLockContextManager:
|
||||
def __init__(self, lock):
|
||||
self._lock = lock
|
||||
|
||||
def __enter__(self):
|
||||
yield self._lock.acquire()
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
yield self._lock.release()
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def DeferredDict(d, consumeErrors=False):
|
||||
keys = []
|
||||
dl = []
|
||||
response = {}
|
||||
for k, v in d.items():
|
||||
keys.append(k)
|
||||
dl.append(v)
|
||||
results = yield defer.DeferredList(dl, consumeErrors=consumeErrors)
|
||||
for k, (success, result) in zip(keys, results):
|
||||
if success:
|
||||
response[k] = result
|
||||
defer.returnValue(response)
|
||||
|
||||
|
||||
class DeferredProfiler:
|
||||
def __init__(self):
|
||||
self.profile_results = {}
|
||||
|
||||
def add_result(self, fn, start_time, finished_time, stack, success):
|
||||
self.profile_results[fn].append((start_time, finished_time, stack, success))
|
||||
|
||||
def show_profile_results(self, fn):
|
||||
profile_results = list(self.profile_results[fn])
|
||||
call_counts = {
|
||||
caller: [(start, finished, finished - start, success)
|
||||
for (start, finished, _caller, success) in profile_results
|
||||
if _caller == caller]
|
||||
for caller in set(result[2] for result in profile_results)
|
||||
}
|
||||
|
||||
log.info("called %s %i times from %i sources\n", fn.__name__, len(profile_results), len(call_counts))
|
||||
for caller in sorted(list(call_counts.keys()), key=lambda c: len(call_counts[c]), reverse=True):
|
||||
call_info = call_counts[caller]
|
||||
times = [r[2] for r in call_info]
|
||||
own_time = sum(times)
|
||||
times.sort()
|
||||
longest = 0 if not times else times[-1]
|
||||
shortest = 0 if not times else times[0]
|
||||
log.info(
|
||||
"%i successes and %i failures\nlongest %f, shortest %f, avg %f\ncaller:\n%s",
|
||||
len([r for r in call_info if r[3]]),
|
||||
len([r for r in call_info if not r[3]]),
|
||||
longest, shortest, own_time / float(len(call_info)), caller
|
||||
)
|
||||
|
||||
def profiled_deferred(self, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
|
||||
def _cb(result, fn, start, caller_info):
|
||||
if isinstance(result, (Failure, Exception)):
|
||||
error = result
|
||||
result = None
|
||||
else:
|
||||
error = None
|
||||
self.add_result(fn, start, reactor.seconds(), caller_info, error is None)
|
||||
if error is None:
|
||||
return result
|
||||
raise error
|
||||
|
||||
def _profiled_deferred(fn):
|
||||
reactor.addSystemEventTrigger("after", "shutdown", self.show_profile_results, fn)
|
||||
self.profile_results[fn] = []
|
||||
|
||||
@functools.wraps(fn)
|
||||
def _wrapper(*args, **kwargs):
|
||||
caller_info = "".join(traceback.format_list(traceback.extract_stack()[-3:-1]))
|
||||
start = reactor.seconds()
|
||||
d = defer.maybeDeferred(fn, *args, **kwargs)
|
||||
d.addBoth(_cb, fn, start, caller_info)
|
||||
return d
|
||||
|
||||
return _wrapper
|
||||
|
||||
return _profiled_deferred
|
||||
|
||||
|
||||
_profiler = DeferredProfiler()
|
||||
profile_deferred = _profiler.profiled_deferred
|
||||