Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e01fbc6f39 | ||
|
|
60ed4e503f | ||
|
|
aa1405887e | ||
|
|
cced217e43 | ||
|
|
8a3b8d2df1 | ||
|
|
be0bd3bdea | ||
|
|
e01c73a7f8 | ||
|
|
134338db4b | ||
|
|
f47a685734 | ||
|
|
6e29f7e99b | ||
|
|
026d93e92a | ||
|
|
a55802c1de | ||
|
|
894397a327 | ||
|
|
3e3f6c0e22 | ||
|
|
a7610e3d34 | ||
|
|
d0e6e74e1b | ||
|
|
126d71978b | ||
|
|
69bcb052de | ||
|
|
071af12d1a | ||
|
|
b7d62ca084 | ||
|
|
fa97f7bd38 | ||
|
|
4a35247016 | ||
|
|
d0526ad7d5 | ||
|
|
19b529cf38 | ||
|
|
143353be44 | ||
|
|
0373196707 | ||
|
|
7a601db23f | ||
|
|
ed61c7feec | ||
|
|
77ece96356 | ||
|
|
80440fd6bd | ||
|
|
ed6537db79 | ||
|
|
79e4767f69 | ||
|
|
0ee279bfd7 | ||
|
|
5249d2a3ca | ||
|
|
4d5823f473 | ||
|
|
c2b9909397 | ||
|
|
afff362544 | ||
|
|
64eb7dbc49 | ||
|
|
2dcd187d69 | ||
|
|
3d9ea92118 | ||
|
|
13a18d422e | ||
|
|
1bbbbd289c | ||
|
|
e33b74f60c | ||
|
|
cd76e0c296 | ||
|
|
3b61222e5a | ||
|
|
2ca438c36e | ||
|
|
8948d5be86 | ||
|
|
a3e64f9cef | ||
|
|
09afec1961 | ||
|
|
f9c465a636 | ||
|
|
89efe3d1b9 | ||
|
|
94cec1b86d | ||
|
|
d62ac0b608 | ||
|
|
5596eac930 | ||
|
|
59f5381b0a | ||
|
|
65ae1b3f08 | ||
|
|
977a7b34f4 | ||
|
|
c01728a6a7 | ||
|
|
5bb94d744e | ||
|
|
494917158c | ||
|
|
ea15674e62 | ||
|
|
34a409ade5 | ||
|
|
273e614715 | ||
|
|
34a725f7ad | ||
|
|
bb6112276f | ||
|
|
db75f0baef | ||
|
|
30689bbf8d | ||
|
|
4bf7742c19 | ||
|
|
69b259c285 | ||
|
|
2fa5233796 | ||
|
|
a5524d490c | ||
|
|
c713fac2d9 | ||
|
|
7a038bbb98 | ||
|
|
eba88c1df7 | ||
|
|
881e5aba68 | ||
|
|
368f0683ec | ||
|
|
94dc8c8520 | ||
|
|
8593cd362a | ||
|
|
2dbf96da81 | ||
|
|
83c7797187 | ||
|
|
6606deafa5 | ||
|
|
19b0d59159 | ||
|
|
9ccb3fa2a3 | ||
|
|
48533df523 | ||
|
|
1ae1137005 | ||
|
|
d5f2832b2f | ||
|
|
d0230b4893 | ||
|
|
a6cd53b97c | ||
|
|
7a28171a72 | ||
|
|
150bcb1116 | ||
|
|
6fe9c5bf00 | ||
|
|
3a86fc6680 | ||
|
|
2805cbb331 | ||
|
|
68d7a7014c | ||
|
|
883fa267a5 | ||
|
|
5f2c3ac965 | ||
|
|
bf8e32ebd2 | ||
|
|
5971be5bef | ||
|
|
677d76a3e4 | ||
|
|
6ed3207cb7 | ||
|
|
be6448b74f | ||
|
|
b707ee4844 | ||
|
|
0e9888be3b | ||
|
|
68b56b7858 | ||
|
|
d32697f611 | ||
|
|
3c54b2bcd7 | ||
|
|
899b182811 | ||
|
|
0dfb5e008d | ||
|
|
abf94357eb | ||
|
|
36aded3830 | ||
|
|
d7ebf50602 | ||
|
|
523c4c86a9 | ||
|
|
74cccdbfc7 | ||
|
|
70471eebfa | ||
|
|
6339224980 | ||
|
|
fe97f7a119 | ||
|
|
f8ed605da2 | ||
|
|
e38ef2af1f | ||
|
|
5d8e627d02 | ||
|
|
c9d4ad5450 | ||
|
|
31586bf588 | ||
|
|
974d3c83e3 | ||
|
|
8dc8ca382b | ||
|
|
1dbb250413 | ||
|
|
9f9ec692c7 | ||
|
|
2c1b54c76c | ||
|
|
7314494fb9 | ||
|
|
e74a5e8ef3 | ||
|
|
b9af999739 | ||
|
|
2de155f90f | ||
|
|
ab3db2e626 | ||
|
|
f9d26b04d8 | ||
|
|
24e0673198 | ||
|
|
833261801f | ||
|
|
6c8cbc1b9e | ||
|
|
6ae4e68d1c | ||
|
|
d22a2dee33 | ||
|
|
0f1ed589a5 | ||
|
|
b032e99dd1 | ||
|
|
fd7811f38d | ||
|
|
071db3a73d | ||
|
|
950c122850 | ||
|
|
e0386ea9dc | ||
|
|
6c3eee0f0e | ||
|
|
4c23add193 | ||
|
|
248d1a18c7 | ||
|
|
17d1b85423 | ||
|
|
b0deb63129 | ||
|
|
ebdb33bd11 | ||
|
|
69ccbda168 | ||
|
|
a38643cca6 | ||
|
|
712b19605e | ||
|
|
8a5f3842fc | ||
|
|
db3e578bcd | ||
|
|
7e22adff7c | ||
|
|
76fbaac658 | ||
|
|
c4fac29ecb | ||
|
|
33392b3351 | ||
|
|
0613cf2d59 | ||
|
|
a930ca0631 | ||
|
|
63a7271008 | ||
|
|
5ca0ebcbb2 | ||
|
|
2c84b7a7dd | ||
|
|
71a0c704bd | ||
|
|
d30e7cd29e | ||
|
|
aeb7c6fe76 | ||
|
|
134c20da93 | ||
|
|
893b4fc3ca | ||
|
|
9b27e81940 | ||
|
|
72c0acf269 | ||
|
|
7be53ad692 | ||
|
|
16e596ec87 | ||
|
|
3fdcb80960 | ||
|
|
26a3221bac | ||
|
|
eb16cb0aa8 | ||
|
|
b5340505e5 | ||
|
|
4bb8129bb7 | ||
|
|
7f838d3a31 | ||
|
|
c186c31bf7 | ||
|
|
86fb99a37c | ||
|
|
6c52450858 | ||
|
|
fb4e94b04a | ||
|
|
248baf58b4 | ||
|
|
0845d65f4e | ||
|
|
dcc5415648 | ||
|
|
3c38a88b22 | ||
|
|
187b7670b1 | ||
|
|
049c17ff08 | ||
|
|
8de97bb441 | ||
|
|
97c0092923 | ||
|
|
e65e1fb5a2 | ||
|
|
20dd1e55f5 | ||
|
|
6aa709e34e | ||
|
|
e73f121c08 | ||
|
|
3b9d6c4f8e | ||
|
|
69b60c19c3 | ||
|
|
04278cb1ab | ||
|
|
e8dec5b440 | ||
|
|
75f82b107e | ||
|
|
0e90c4b611 | ||
|
|
fb66b68bc8 | ||
|
|
c601677386 | ||
|
|
1fbd1cd04e | ||
|
|
4a71b91fbf | ||
|
|
b8b0d8b1ac | ||
|
|
8ef73d3985 | ||
|
|
05e556a5e3 | ||
|
|
d1da42bea8 | ||
|
|
90685e8357 | ||
|
|
a471d5d167 | ||
|
|
16122299e4 | ||
|
|
9f18da4bae | ||
|
|
1abce52867 | ||
|
|
072e2b65d3 | ||
|
|
f6d11e5467 | ||
|
|
458bddcb65 | ||
|
|
5fad43ff69 | ||
|
|
6992d0d947 | ||
|
|
d1b0180437 | ||
|
|
ba59e126cd | ||
|
|
4e3467bb7e | ||
|
|
4d4c705eb9 | ||
|
|
ce7b3184c4 | ||
|
|
b71faddaac | ||
|
|
1328cb4cce | ||
|
|
946488bce9 | ||
|
|
7713881e71 | ||
|
|
26fcc48c5a | ||
|
|
bf12fb68a2 | ||
|
|
3a4c9ff4f9 | ||
|
|
6180a2917d | ||
|
|
014cbb4623 | ||
|
|
ee98251d79 | ||
|
|
e3d08ec773 | ||
|
|
f03e68e63e | ||
|
|
72090b45dd | ||
|
|
93f9622798 | ||
|
|
16ff4b4e2c | ||
|
|
c90cca4b20 | ||
|
|
98d83ea74f | ||
|
|
a234f92d2c | ||
|
|
8ba3575901 | ||
|
|
a857c83c59 | ||
|
|
a6128ac8a2 | ||
|
|
2693301ad0 | ||
|
|
f7c7cb9535 | ||
|
|
b36a5f9b6b | ||
|
|
4826eb9969 | ||
|
|
c34b150794 | ||
|
|
c4c7670a7c | ||
|
|
0918bac86e | ||
|
|
a60a921706 | ||
|
|
b3ee29b66d | ||
|
|
3a1a9b988c | ||
|
|
12f2bebc46 | ||
|
|
31a3812c8a | ||
|
|
70ec08e30b | ||
|
|
23e8f0d849 | ||
|
|
1cdfcd5b4e | ||
|
|
dc2786fbc0 | ||
|
|
7e39b0c4fe | ||
|
|
4af830c405 | ||
|
|
fe844581fd | ||
|
|
443ce6352b | ||
|
|
158c354a27 | ||
|
|
92d1230a90 | ||
|
|
d64916a061 | ||
|
|
20a0568253 | ||
|
|
5a1f42ee54 | ||
|
|
ab95f14e21 | ||
|
|
bae2676b69 | ||
|
|
ea6869a63e | ||
|
|
953327c150 | ||
|
|
e01ad6bc85 | ||
|
|
47a33009ff | ||
|
|
bc844013a1 | ||
|
|
2fd2319cdb | ||
|
|
a50657c472 | ||
|
|
ef5f6e06ba | ||
|
|
977cf9f506 | ||
|
|
ffae10d11f | ||
|
|
221107e76e | ||
|
|
e74c78b607 | ||
|
|
434c7b57b1 | ||
|
|
9dc4608a62 | ||
|
|
fc69366092 | ||
|
|
9f0b1f3e25 | ||
|
|
3f03a845ec | ||
|
|
bc28976e12 | ||
|
|
8f826098f9 | ||
|
|
0a765e6207 | ||
|
|
f83aa61923 | ||
|
|
f9ff3e5d14 | ||
|
|
b3fde9d78d | ||
|
|
0c999083cc | ||
|
|
561215db8a | ||
|
|
8dbb7f2c78 | ||
|
|
6f7f3bb316 | ||
|
|
8d507c4b73 | ||
|
|
2beb0eda94 | ||
|
|
b7e07b72ca | ||
|
|
fde8718c8c | ||
|
|
b2d1bddcfe | ||
|
|
dd1d1be994 | ||
|
|
216c760179 | ||
|
|
01eca3004f |
No files matched your search
@@ -0,0 +1,7 @@
|
||||
.git
|
||||
.tox
|
||||
__pycache__
|
||||
dist
|
||||
lbrynet.egg-info
|
||||
docs
|
||||
tests
|
||||
@@ -1,15 +1,38 @@
|
||||
<!--
|
||||
Thank your for making LBRY better!
|
||||
## PR Checklist
|
||||
Please check all that apply to this PR using "x":
|
||||
|
||||
note: Check existing issues and pull-requests before submitting new ones, as a courtesy to the maintainers and making sure work isn't duplicated.
|
||||
-->
|
||||
### Description
|
||||
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
|
||||
- [ ] I have checked that this PR does not introduce a breaking change
|
||||
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
|
||||
|
||||
<!-- Please describe what the changes are made in this pull request and why the change was necessary. -->
|
||||
|
||||
<!--
|
||||
If this pull request applies to an issue, add in the issue number here
|
||||
e.g.
|
||||
Fixes #1
|
||||
-->
|
||||
### Fixes #
|
||||
## PR Type
|
||||
What kind of change does this PR introduce?
|
||||
|
||||
Why is this change necessary?
|
||||
|
||||
<!-- Please check all that apply to this PR using "x". -->
|
||||
|
||||
- [ ] Bugfix
|
||||
- [ ] Feature
|
||||
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
|
||||
- [ ] Code style update (formatting)
|
||||
- [ ] Refactoring (no functional changes)
|
||||
- [ ] Documentation changes
|
||||
- [ ] Other - Please describe:
|
||||
|
||||
## Fixes
|
||||
|
||||
Issue Number: N/A
|
||||
|
||||
|
||||
## What is the current behavior?
|
||||
|
||||
|
||||
## What is the new behavior?
|
||||
|
||||
|
||||
## Other information
|
||||
|
||||
|
||||
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
|
||||
@@ -354,13 +354,13 @@ ignore-mixin-members=yes
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=twisted.internet.reactor,leveldb,distutils
|
||||
ignored-modules=leveldb,distutils
|
||||
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
|
||||
|
||||
# List of classes names for which member attributes should not be checked
|
||||
# (useful for classes with attributes dynamically set). This supports can work
|
||||
# with qualified names.
|
||||
ignored-classes=twisted.internet.reactor,RequestMessage
|
||||
# ignored-classes=
|
||||
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
|
||||
@@ -9,38 +9,25 @@ jobs:
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid==2.0.4 aiohttp==3.4.4
|
||||
# newer astroid and aiohttp fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba[server]
|
||||
- pip install -e .[wallet-server]
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script: pylint lbrynet
|
||||
|
||||
- &tests
|
||||
stage: test
|
||||
name: "Unit Tests w/ Python 3.7"
|
||||
- stage: test
|
||||
name: "Unit Tests"
|
||||
install:
|
||||
- pip install coverage
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba[server]
|
||||
- pip install -e .[test]
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -v tests.unit.wallet
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics tests.unit.components tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.test_cli tests.unit.test_customLogger
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- <<: *tests
|
||||
name: "Unit Tests w/ Python 3.6"
|
||||
python: "3.6"
|
||||
|
||||
- <<: *tests
|
||||
name: "DHT Tests w/ Python 3.7"
|
||||
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
|
||||
|
||||
- <<: *tests
|
||||
name: "DHT Tests w/ Python 3.6"
|
||||
python: "3.6"
|
||||
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
|
||||
|
||||
- name: "Integration Tests"
|
||||
install:
|
||||
- pip install tox-travis coverage
|
||||
@@ -56,21 +43,31 @@ jobs:
|
||||
services:
|
||||
- docker
|
||||
install:
|
||||
- docker pull cdrx/pyinstaller-windows:python3-32bit
|
||||
- docker pull lbry/pyinstaller34_32bits:py371
|
||||
script:
|
||||
- docker run -v "$(pwd):/src/lbry" cdrx/pyinstaller-windows:python3-32bit lbry/scripts/wine_build.sh
|
||||
- python scripts/set_build.py
|
||||
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
|
||||
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-windows.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
paths:
|
||||
- lbrynet-windows.zip
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})/win/
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- &build
|
||||
name: "Linux"
|
||||
python: "3.6"
|
||||
env: OS=linux
|
||||
install:
|
||||
- pip3 install pyinstaller
|
||||
- pip3 install git+https://github.com/lbryio/torba.git
|
||||
@@ -81,7 +78,15 @@ jobs:
|
||||
- chmod +x dist/lbrynet
|
||||
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
|
||||
- ./dist/lbrynet --version
|
||||
env: OS=linux
|
||||
deploy:
|
||||
provider: releases
|
||||
api_key: $GITHUB_OAUTH_TOKEN
|
||||
file: dist/lbrynet-${OS}.zip
|
||||
skip_cleanup: true
|
||||
overwrite: true
|
||||
draft: true
|
||||
on:
|
||||
tags: true
|
||||
addons:
|
||||
artifacts:
|
||||
working_dir: dist
|
||||
@@ -89,14 +94,44 @@ jobs:
|
||||
- lbrynet-${OS}.zip
|
||||
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
|
||||
target_paths:
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})/${OS}/lbrynet
|
||||
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
|
||||
|
||||
- <<: *build
|
||||
name: "Mac"
|
||||
os: osx
|
||||
osx_image: xcode9.4
|
||||
osx_image: xcode8.3
|
||||
language: generic
|
||||
env: OS=mac
|
||||
cache: false
|
||||
before_install:
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
- pip3 install pyinstaller
|
||||
- git clone https://github.com/lbryio/torba.git --depth 1
|
||||
- sed -i -e "s/'plyvel',//" torba/setup.py
|
||||
- cd torba && pip3 install -e . && cd ..
|
||||
- python3 scripts/set_build.py
|
||||
- pip3 install -e .
|
||||
|
||||
- if: tag IS present
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Tagged Release"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:$TRAVIS_TAG
|
||||
- if: tag IS blank AND branch = master
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Master"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:master
|
||||
|
||||
cache:
|
||||
directories:
|
||||
|
||||
@@ -8,44 +8,44 @@ can and probably will change functionality and break backwards compatibility
|
||||
at anytime.
|
||||
|
||||
|
||||
## Unreleased
|
||||
|
||||
### Security
|
||||
*
|
||||
*
|
||||
## [0.30.1] - 2018-12-09
|
||||
|
||||
### API
|
||||
* add `aioupnp_version` to `upnp` field in the response to `status`
|
||||
* fix uncaught claim signature errors
|
||||
* fixed uncaught claim signature errors
|
||||
* `--blocking` support for `abandon` command (https://github.com/lbryio/lbry/pull/1618)
|
||||
* don't create SSL keys if SSL is disabled (https://github.com/lbryio/lbry/pull/1644)
|
||||
* added `account_send` command (https://github.com/lbryio/lbry/pull/1654)
|
||||
* fixes for claim send to address (https://github.com/lbryio/lbry/pull/1651)
|
||||
* better handle errors on resolve (https://github.com/lbryio/lbry/pull/1662)
|
||||
* fix `transaction_show` (https://github.com/lbryio/lbry/pull/1689)
|
||||
|
||||
### Schema
|
||||
* merged `lbryschema` repository into `lbrynet` (https://github.com/lbryio/lbry/pull/1444)
|
||||
* updated `protobuf` to `3.2.0` (https://github.com/lbryio/lbry/issues/1602)
|
||||
* use `types` repo for generating python protobufs (https://github.com/lbryio/lbry/pull/1605)
|
||||
|
||||
### Wallet
|
||||
*
|
||||
*
|
||||
|
||||
### File Manager
|
||||
*
|
||||
*
|
||||
* fixed resolving names with invalid channel signatures
|
||||
* fixed wallet sync issues
|
||||
* fixed wallet locking/encryption issues
|
||||
|
||||
### DHT
|
||||
*
|
||||
*
|
||||
|
||||
### P2P
|
||||
*
|
||||
*
|
||||
|
||||
### Reflector
|
||||
*
|
||||
*
|
||||
* fixed retrying kademlia rpc calls (https://github.com/lbryio/lbry/pull/1495)
|
||||
* fixed `KeyError` in `iterativeFindValue`, this is used by `peer_list` and when finding peers to download blobs from (https://github.com/lbryio/lbry/issues/1638)
|
||||
* settings external ip address and ability to disable more DHT components (https://github.com/lbryio/lbry/pull/1604)
|
||||
* improved routing table bucket initialization (https://github.com/lbryio/lbry/pull/1633)
|
||||
|
||||
### Database
|
||||
*
|
||||
*
|
||||
|
||||
### Known Issues
|
||||
*
|
||||
*
|
||||
* resolved too many variables in sqlite issue (https://github.com/lbryio/lbry/pull/1612)
|
||||
|
||||
### Misc
|
||||
* switch to Python 3.7
|
||||
* brought back icon for windows binary
|
||||
* using custom list of mime types with more types (https://github.com/lbryio/lbry/pull/1700)
|
||||
* add balance to startup report (https://github.com/lbryio/lbry/pull/1613)
|
||||
* drop `six` dependency (https://github.com/lbryio/lbry/pull/1660)
|
||||
* drop `mock` dependency (https://github.com/lbryio/lbry/pull/1661)
|
||||
|
||||
## [0.30.0] - 2018-10-22
|
||||
This release is the most ambitious upgrade to lbrynet yet; including a brand
|
||||
|
||||
@@ -57,7 +57,7 @@ To install:
|
||||
|
||||
python --version # Python 2 is not supported. Make sure you're on Python 3.7
|
||||
|
||||
pip install --editable .[test] # [test] installs extras needed for running the tests
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized peer-to-peer network. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
This project provides an SDK for building applications using the LBRY protocol. The SDK is written in Python 3.7+ using Twisted. For other languages or platforms, a JSON-RPC API is provided.
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -38,7 +45,7 @@ We take security seriously. Please contact security@lbry.io regarding any securi
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@jackrobison](mailto:jack@lbry.io).
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
build:
|
||||
python setup.py apk
|
||||
adb install -r lbry-0.1-service-debug.apk
|
||||
|
||||
.PHONY: build
|
||||
@@ -0,0 +1,49 @@
|
||||
import inspect
|
||||
from os.path import dirname, join
|
||||
|
||||
import sh
|
||||
from pythonforandroid.util import ensure_dir
|
||||
from pythonforandroid.toolchain import (
|
||||
Bootstrap, current_directory, info, info_main, shprint
|
||||
)
|
||||
|
||||
|
||||
class LBRYServiceBootstrap(Bootstrap):
|
||||
name = 'lbry-service'
|
||||
recipe_depends = ['genericndkbuild', 'python3']
|
||||
bootstrap_dir = dirname(__file__)
|
||||
|
||||
def get_common_dir(self):
|
||||
return join(dirname(inspect.getfile(Bootstrap)), 'bootstraps', 'common')
|
||||
|
||||
def run_distribute(self):
|
||||
info_main('# Creating Android project from build and {} bootstrap'.format(
|
||||
self.name))
|
||||
|
||||
info('This currently just copies the build stuff straight from the build dir.')
|
||||
shprint(sh.rm, '-rf', self.dist_dir)
|
||||
shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
|
||||
with current_directory(self.dist_dir):
|
||||
with open('local.properties', 'w') as fileh:
|
||||
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
|
||||
|
||||
arch = self.ctx.archs[0]
|
||||
if len(self.ctx.archs) > 1:
|
||||
raise ValueError('built for more than one arch, but bootstrap cannot handle that yet')
|
||||
info('Bootstrap running with arch {}'.format(arch))
|
||||
|
||||
with current_directory(self.dist_dir):
|
||||
info('Copying python distribution')
|
||||
|
||||
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
|
||||
self.distribute_aars(arch)
|
||||
self.distribute_javaclasses(self.ctx.javaclass_dir)
|
||||
|
||||
python_bundle_dir = join('_python_bundle', '_python_bundle')
|
||||
ensure_dir(python_bundle_dir)
|
||||
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
|
||||
join(self.dist_dir, python_bundle_dir), arch)
|
||||
|
||||
self.strip_libraries(arch)
|
||||
self.fry_eggs(site_packages_dir)
|
||||
super().run_distribute()
|
||||
@@ -0,0 +1,89 @@
|
||||
# prevent user to include invalid extensions
|
||||
*.apk
|
||||
*.pxd
|
||||
|
||||
# eggs
|
||||
*.egg-info
|
||||
|
||||
# unit test
|
||||
unittest/*
|
||||
|
||||
# python config
|
||||
config/makesetup
|
||||
|
||||
# unused kivy files (platform specific)
|
||||
kivy/input/providers/wm_*
|
||||
kivy/input/providers/mactouch*
|
||||
kivy/input/providers/probesysfs*
|
||||
kivy/input/providers/mtdev*
|
||||
kivy/input/providers/hidinput*
|
||||
kivy/core/camera/camera_videocapture*
|
||||
kivy/core/spelling/*osx*
|
||||
kivy/core/video/video_pyglet*
|
||||
kivy/tools
|
||||
kivy/tests/*
|
||||
kivy/*/*.h
|
||||
kivy/*/*.pxi
|
||||
|
||||
# unused encodings
|
||||
lib-dynload/*codec*
|
||||
encodings/cp*.pyo
|
||||
encodings/tis*
|
||||
encodings/shift*
|
||||
encodings/bz2*
|
||||
encodings/iso*
|
||||
encodings/undefined*
|
||||
encodings/johab*
|
||||
encodings/p*
|
||||
encodings/m*
|
||||
encodings/euc*
|
||||
encodings/k*
|
||||
encodings/unicode_internal*
|
||||
encodings/quo*
|
||||
encodings/gb*
|
||||
encodings/big5*
|
||||
encodings/hp*
|
||||
encodings/hz*
|
||||
|
||||
# unused python modules
|
||||
bsddb/*
|
||||
wsgiref/*
|
||||
hotshot/*
|
||||
pydoc_data/*
|
||||
tty.pyo
|
||||
anydbm.pyo
|
||||
nturl2path.pyo
|
||||
LICENCE.txt
|
||||
macurl2path.pyo
|
||||
dummy_threading.pyo
|
||||
audiodev.pyo
|
||||
antigravity.pyo
|
||||
dumbdbm.pyo
|
||||
sndhdr.pyo
|
||||
__phello__.foo.pyo
|
||||
sunaudio.pyo
|
||||
os2emxpath.pyo
|
||||
multiprocessing/dummy*
|
||||
|
||||
# unused binaries python modules
|
||||
lib-dynload/termios.so
|
||||
lib-dynload/_lsprof.so
|
||||
lib-dynload/*audioop.so
|
||||
lib-dynload/_hotshot.so
|
||||
lib-dynload/_heapq.so
|
||||
lib-dynload/_json.so
|
||||
lib-dynload/grp.so
|
||||
lib-dynload/resource.so
|
||||
lib-dynload/pyexpat.so
|
||||
lib-dynload/_ctypes_test.so
|
||||
lib-dynload/_testcapi.so
|
||||
|
||||
# odd files
|
||||
plat-linux3/regen
|
||||
|
||||
#>sqlite3
|
||||
# conditionnal include depending if some recipes are included or not.
|
||||
sqlite3/*
|
||||
lib-dynload/_sqlite3.so
|
||||
#<sqlite3
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
include $(call all-subdir-makefiles)
|
||||
@@ -0,0 +1,7 @@
|
||||
|
||||
# Uncomment this if you're using STL in your project
|
||||
# See CPLUSPLUS-SUPPORT.html in the NDK documentation for more information
|
||||
# APP_STL := stlport_static
|
||||
|
||||
# APP_ABI := armeabi armeabi-v7a x86
|
||||
APP_ABI := $(ARCH)
|
||||
@@ -0,0 +1,22 @@
|
||||
LOCAL_PATH := $(call my-dir)
|
||||
|
||||
include $(CLEAR_VARS)
|
||||
|
||||
LOCAL_MODULE := main
|
||||
|
||||
# Add your application source files here...
|
||||
LOCAL_SRC_FILES := start.c pyjniusjni.c
|
||||
|
||||
LOCAL_CFLAGS += -I$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_INCLUDE_ROOT) $(EXTRA_CFLAGS)
|
||||
|
||||
LOCAL_SHARED_LIBRARIES := python_shared
|
||||
|
||||
LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS)
|
||||
|
||||
LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS)
|
||||
|
||||
include $(BUILD_SHARED_LIBRARY)
|
||||
|
||||
ifdef CRYSTAX_PYTHON_VERSION
|
||||
$(call import-module,python/$(CRYSTAX_PYTHON_VERSION))
|
||||
endif
|
||||
@@ -0,0 +1,10 @@
|
||||
LOCAL_PATH := $(call my-dir)
|
||||
|
||||
include $(CLEAR_VARS)
|
||||
|
||||
LOCAL_MODULE := main
|
||||
|
||||
LOCAL_SRC_FILES := YourSourceHere.c
|
||||
|
||||
include $(BUILD_SHARED_LIBRARY)
|
||||
$(call import-module,SDL)LOCAL_PATH := $(call my-dir)
|
||||
@@ -0,0 +1,6 @@
|
||||
|
||||
#define BOOTSTRAP_NAME_SERVICEONLY
|
||||
#define BOOTSTRAP_USES_NO_SDL_HEADERS
|
||||
|
||||
const char bootstrap_name[] = "service_only";
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
|
||||
#include <pthread.h>
|
||||
#include <jni.h>
|
||||
|
||||
#define LOGI(...) do {} while (0)
|
||||
#define LOGE(...) do {} while (0)
|
||||
|
||||
#include "android/log.h"
|
||||
|
||||
/* These JNI management functions are taken from SDL2, but modified to refer to pyjnius */
|
||||
|
||||
/* #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) */
|
||||
/* #define LOGP(x) LOG("python", (x)) */
|
||||
#define LOG_TAG "Python_android"
|
||||
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
|
||||
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
|
||||
|
||||
|
||||
/* Function headers */
|
||||
JNIEnv* Android_JNI_GetEnv(void);
|
||||
static void Android_JNI_ThreadDestroyed(void*);
|
||||
|
||||
static pthread_key_t mThreadKey;
|
||||
static JavaVM* mJavaVM;
|
||||
|
||||
int Android_JNI_SetupThread(void)
|
||||
{
|
||||
Android_JNI_GetEnv();
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Library init */
|
||||
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
|
||||
{
|
||||
JNIEnv *env;
|
||||
mJavaVM = vm;
|
||||
LOGI("JNI_OnLoad called");
|
||||
if ((*mJavaVM)->GetEnv(mJavaVM, (void**) &env, JNI_VERSION_1_4) != JNI_OK) {
|
||||
LOGE("Failed to get the environment using GetEnv()");
|
||||
return -1;
|
||||
}
|
||||
/*
|
||||
* Create mThreadKey so we can keep track of the JNIEnv assigned to each thread
|
||||
* Refer to http://developer.android.com/guide/practices/design/jni.html for the rationale behind this
|
||||
*/
|
||||
if (pthread_key_create(&mThreadKey, Android_JNI_ThreadDestroyed) != 0) {
|
||||
|
||||
__android_log_print(ANDROID_LOG_ERROR, "pyjniusjni", "Error initializing pthread key");
|
||||
}
|
||||
Android_JNI_SetupThread();
|
||||
|
||||
return JNI_VERSION_1_4;
|
||||
}
|
||||
|
||||
JNIEnv* Android_JNI_GetEnv(void)
|
||||
{
|
||||
/* From http://developer.android.com/guide/practices/jni.html
|
||||
* All threads are Linux threads, scheduled by the kernel.
|
||||
* They're usually started from managed code (using Thread.start), but they can also be created elsewhere and then
|
||||
* attached to the JavaVM. For example, a thread started with pthread_create can be attached with the
|
||||
* JNI AttachCurrentThread or AttachCurrentThreadAsDaemon functions. Until a thread is attached, it has no JNIEnv,
|
||||
* and cannot make JNI calls.
|
||||
* Attaching a natively-created thread causes a java.lang.Thread object to be constructed and added to the "main"
|
||||
* ThreadGroup, making it visible to the debugger. Calling AttachCurrentThread on an already-attached thread
|
||||
* is a no-op.
|
||||
* Note: You can call this function any number of times for the same thread, there's no harm in it
|
||||
*/
|
||||
|
||||
JNIEnv *env;
|
||||
int status = (*mJavaVM)->AttachCurrentThread(mJavaVM, &env, NULL);
|
||||
if(status < 0) {
|
||||
LOGE("failed to attach current thread");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* From http://developer.android.com/guide/practices/jni.html
|
||||
* Threads attached through JNI must call DetachCurrentThread before they exit. If coding this directly is awkward,
|
||||
* in Android 2.0 (Eclair) and higher you can use pthread_key_create to define a destructor function that will be
|
||||
* called before the thread exits, and call DetachCurrentThread from there. (Use that key with pthread_setspecific
|
||||
* to store the JNIEnv in thread-local-storage; that way it'll be passed into your destructor as the argument.)
|
||||
* Note: The destructor is not called unless the stored value is != NULL
|
||||
* Note: You can call this function any number of times for the same thread, there's no harm in it
|
||||
* (except for some lost CPU cycles)
|
||||
*/
|
||||
pthread_setspecific(mThreadKey, (void*) env);
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
static void Android_JNI_ThreadDestroyed(void* value)
|
||||
{
|
||||
/* The thread is being destroyed, detach it from the Java VM and set the mThreadKey value to NULL as required */
|
||||
JNIEnv *env = (JNIEnv*) value;
|
||||
if (env != NULL) {
|
||||
(*mJavaVM)->DetachCurrentThread(mJavaVM);
|
||||
pthread_setspecific(mThreadKey, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
void *WebView_AndroidGetJNIEnv()
|
||||
{
|
||||
return Android_JNI_GetEnv();
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class Octal {
|
||||
|
||||
/**
|
||||
* Parse an octal string from a header buffer. This is used for the file
|
||||
* permission mode value.
|
||||
*
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The long value of the octal string.
|
||||
*/
|
||||
public static long parseOctal(byte[] header, int offset, int length) {
|
||||
long result = 0;
|
||||
boolean stillPadding = true;
|
||||
|
||||
int end = offset + length;
|
||||
for (int i = offset; i < end; ++i) {
|
||||
if (header[i] == 0)
|
||||
break;
|
||||
|
||||
if (header[i] == (byte) ' ' || header[i] == '0') {
|
||||
if (stillPadding)
|
||||
continue;
|
||||
|
||||
if (header[i] == (byte) ' ')
|
||||
break;
|
||||
}
|
||||
|
||||
stillPadding = false;
|
||||
|
||||
result = ( result << 3 ) + ( header[i] - '0' );
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an octal integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The integer value of the octal bytes.
|
||||
*/
|
||||
public static int getOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
int idx = length - 1;
|
||||
|
||||
buf[offset + idx] = 0;
|
||||
--idx;
|
||||
buf[offset + idx] = (byte) ' ';
|
||||
--idx;
|
||||
|
||||
if (value == 0) {
|
||||
buf[offset + idx] = (byte) '0';
|
||||
--idx;
|
||||
} else {
|
||||
for (long val = value; idx >= 0 && val > 0; --idx) {
|
||||
buf[offset + idx] = (byte) ( (byte) '0' + (byte) ( val & 7 ) );
|
||||
val = val >> 3;
|
||||
}
|
||||
}
|
||||
|
||||
for (; idx >= 0; --idx) {
|
||||
buf[offset + idx] = (byte) ' ';
|
||||
}
|
||||
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the checksum octal integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The integer value of the entry's checksum.
|
||||
*/
|
||||
public static int getCheckSumOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
getOctalBytes( value, buf, offset, length );
|
||||
buf[offset + length - 1] = (byte) ' ';
|
||||
buf[offset + length - 2] = 0;
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an octal long integer from a header buffer.
|
||||
*
|
||||
* @param value
|
||||
* @param buf
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
*
|
||||
* @return The long value of the octal bytes.
|
||||
*/
|
||||
public static int getLongOctalBytes(long value, byte[] buf, int offset, int length) {
|
||||
byte[] temp = new byte[length + 1];
|
||||
getOctalBytes( value, temp, 0, length + 1 );
|
||||
System.arraycopy( temp, 0, buf, offset, length );
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarConstants {
|
||||
public static final int EOF_BLOCK = 1024;
|
||||
public static final int DATA_BLOCK = 512;
|
||||
public static final int HEADER_BLOCK = 512;
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarEntry {
|
||||
protected File file;
|
||||
protected TarHeader header;
|
||||
|
||||
private TarEntry() {
|
||||
this.file = null;
|
||||
header = new TarHeader();
|
||||
}
|
||||
|
||||
public TarEntry(File file, String entryName) {
|
||||
this();
|
||||
this.file = file;
|
||||
this.extractTarHeader(entryName);
|
||||
}
|
||||
|
||||
public TarEntry(byte[] headerBuf) {
|
||||
this();
|
||||
this.parseTarHeader(headerBuf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructor to create an entry from an existing TarHeader object.
|
||||
*
|
||||
* This method is useful to add new entries programmatically (e.g. for
|
||||
* adding files or directories that do not exist in the file system).
|
||||
*
|
||||
* @param header
|
||||
*
|
||||
*/
|
||||
public TarEntry(TarHeader header) {
|
||||
this.file = null;
|
||||
this.header = header;
|
||||
}
|
||||
|
||||
public boolean equals(TarEntry it) {
|
||||
return header.name.toString().equals(it.header.name.toString());
|
||||
}
|
||||
|
||||
public boolean isDescendent(TarEntry desc) {
|
||||
return desc.header.name.toString().startsWith(header.name.toString());
|
||||
}
|
||||
|
||||
public TarHeader getHeader() {
|
||||
return header;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
String name = header.name.toString();
|
||||
if (header.namePrefix != null && !header.namePrefix.toString().equals("")) {
|
||||
name = header.namePrefix.toString() + "/" + name;
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
header.name = new StringBuffer(name);
|
||||
}
|
||||
|
||||
public int getUserId() {
|
||||
return header.userId;
|
||||
}
|
||||
|
||||
public void setUserId(int userId) {
|
||||
header.userId = userId;
|
||||
}
|
||||
|
||||
public int getGroupId() {
|
||||
return header.groupId;
|
||||
}
|
||||
|
||||
public void setGroupId(int groupId) {
|
||||
header.groupId = groupId;
|
||||
}
|
||||
|
||||
public String getUserName() {
|
||||
return header.userName.toString();
|
||||
}
|
||||
|
||||
public void setUserName(String userName) {
|
||||
header.userName = new StringBuffer(userName);
|
||||
}
|
||||
|
||||
public String getGroupName() {
|
||||
return header.groupName.toString();
|
||||
}
|
||||
|
||||
public void setGroupName(String groupName) {
|
||||
header.groupName = new StringBuffer(groupName);
|
||||
}
|
||||
|
||||
public void setIds(int userId, int groupId) {
|
||||
this.setUserId(userId);
|
||||
this.setGroupId(groupId);
|
||||
}
|
||||
|
||||
public void setModTime(long time) {
|
||||
header.modTime = time / 1000;
|
||||
}
|
||||
|
||||
public void setModTime(Date time) {
|
||||
header.modTime = time.getTime() / 1000;
|
||||
}
|
||||
|
||||
public Date getModTime() {
|
||||
return new Date(header.modTime * 1000);
|
||||
}
|
||||
|
||||
public File getFile() {
|
||||
return this.file;
|
||||
}
|
||||
|
||||
public long getSize() {
|
||||
return header.size;
|
||||
}
|
||||
|
||||
public void setSize(long size) {
|
||||
header.size = size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the org.kamrazafar.jtar entry is a directory
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public boolean isDirectory() {
|
||||
if (this.file != null)
|
||||
return this.file.isDirectory();
|
||||
|
||||
if (header != null) {
|
||||
if (header.linkFlag == TarHeader.LF_DIR)
|
||||
return true;
|
||||
|
||||
if (header.name.toString().endsWith("/"))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract header from File
|
||||
*
|
||||
* @param entryName
|
||||
*/
|
||||
public void extractTarHeader(String entryName) {
|
||||
header = TarHeader.createHeader(entryName, file.length(), file.lastModified() / 1000, file.isDirectory());
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate checksum
|
||||
*
|
||||
* @param buf
|
||||
* @return
|
||||
*/
|
||||
public long computeCheckSum(byte[] buf) {
|
||||
long sum = 0;
|
||||
|
||||
for (int i = 0; i < buf.length; ++i) {
|
||||
sum += 255 & buf[i];
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the header to the byte buffer
|
||||
*
|
||||
* @param outbuf
|
||||
*/
|
||||
public void writeEntryHeader(byte[] outbuf) {
|
||||
int offset = 0;
|
||||
|
||||
offset = TarHeader.getNameBytes(header.name, outbuf, offset, TarHeader.NAMELEN);
|
||||
offset = Octal.getOctalBytes(header.mode, outbuf, offset, TarHeader.MODELEN);
|
||||
offset = Octal.getOctalBytes(header.userId, outbuf, offset, TarHeader.UIDLEN);
|
||||
offset = Octal.getOctalBytes(header.groupId, outbuf, offset, TarHeader.GIDLEN);
|
||||
|
||||
long size = header.size;
|
||||
|
||||
offset = Octal.getLongOctalBytes(size, outbuf, offset, TarHeader.SIZELEN);
|
||||
offset = Octal.getLongOctalBytes(header.modTime, outbuf, offset, TarHeader.MODTIMELEN);
|
||||
|
||||
int csOffset = offset;
|
||||
for (int c = 0; c < TarHeader.CHKSUMLEN; ++c)
|
||||
outbuf[offset++] = (byte) ' ';
|
||||
|
||||
outbuf[offset++] = header.linkFlag;
|
||||
|
||||
offset = TarHeader.getNameBytes(header.linkName, outbuf, offset, TarHeader.NAMELEN);
|
||||
offset = TarHeader.getNameBytes(header.magic, outbuf, offset, TarHeader.USTAR_MAGICLEN);
|
||||
offset = TarHeader.getNameBytes(header.userName, outbuf, offset, TarHeader.USTAR_USER_NAMELEN);
|
||||
offset = TarHeader.getNameBytes(header.groupName, outbuf, offset, TarHeader.USTAR_GROUP_NAMELEN);
|
||||
offset = Octal.getOctalBytes(header.devMajor, outbuf, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset = Octal.getOctalBytes(header.devMinor, outbuf, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset = TarHeader.getNameBytes(header.namePrefix, outbuf, offset, TarHeader.USTAR_FILENAME_PREFIX);
|
||||
|
||||
for (; offset < outbuf.length;)
|
||||
outbuf[offset++] = 0;
|
||||
|
||||
long checkSum = this.computeCheckSum(outbuf);
|
||||
|
||||
Octal.getCheckSumOctalBytes(checkSum, outbuf, csOffset, TarHeader.CHKSUMLEN);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the tar header to the byte buffer
|
||||
*
|
||||
* @param header
|
||||
* @param bh
|
||||
*/
|
||||
public void parseTarHeader(byte[] bh) {
|
||||
int offset = 0;
|
||||
|
||||
header.name = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
|
||||
offset += TarHeader.NAMELEN;
|
||||
|
||||
header.mode = (int) Octal.parseOctal(bh, offset, TarHeader.MODELEN);
|
||||
offset += TarHeader.MODELEN;
|
||||
|
||||
header.userId = (int) Octal.parseOctal(bh, offset, TarHeader.UIDLEN);
|
||||
offset += TarHeader.UIDLEN;
|
||||
|
||||
header.groupId = (int) Octal.parseOctal(bh, offset, TarHeader.GIDLEN);
|
||||
offset += TarHeader.GIDLEN;
|
||||
|
||||
header.size = Octal.parseOctal(bh, offset, TarHeader.SIZELEN);
|
||||
offset += TarHeader.SIZELEN;
|
||||
|
||||
header.modTime = Octal.parseOctal(bh, offset, TarHeader.MODTIMELEN);
|
||||
offset += TarHeader.MODTIMELEN;
|
||||
|
||||
header.checkSum = (int) Octal.parseOctal(bh, offset, TarHeader.CHKSUMLEN);
|
||||
offset += TarHeader.CHKSUMLEN;
|
||||
|
||||
header.linkFlag = bh[offset++];
|
||||
|
||||
header.linkName = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
|
||||
offset += TarHeader.NAMELEN;
|
||||
|
||||
header.magic = TarHeader.parseName(bh, offset, TarHeader.USTAR_MAGICLEN);
|
||||
offset += TarHeader.USTAR_MAGICLEN;
|
||||
|
||||
header.userName = TarHeader.parseName(bh, offset, TarHeader.USTAR_USER_NAMELEN);
|
||||
offset += TarHeader.USTAR_USER_NAMELEN;
|
||||
|
||||
header.groupName = TarHeader.parseName(bh, offset, TarHeader.USTAR_GROUP_NAMELEN);
|
||||
offset += TarHeader.USTAR_GROUP_NAMELEN;
|
||||
|
||||
header.devMajor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset += TarHeader.USTAR_DEVLEN;
|
||||
|
||||
header.devMinor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
|
||||
offset += TarHeader.USTAR_DEVLEN;
|
||||
|
||||
header.namePrefix = TarHeader.parseName(bh, offset, TarHeader.USTAR_FILENAME_PREFIX);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* Header
|
||||
*
|
||||
* <pre>
|
||||
* Offset Size Field
|
||||
* 0 100 File name
|
||||
* 100 8 File mode
|
||||
* 108 8 Owner's numeric user ID
|
||||
* 116 8 Group's numeric user ID
|
||||
* 124 12 File size in bytes
|
||||
* 136 12 Last modification time in numeric Unix time format
|
||||
* 148 8 Checksum for header block
|
||||
* 156 1 Link indicator (file type)
|
||||
* 157 100 Name of linked file
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
* File Types
|
||||
*
|
||||
* <pre>
|
||||
* Value Meaning
|
||||
* '0' Normal file
|
||||
* (ASCII NUL) Normal file (now obsolete)
|
||||
* '1' Hard link
|
||||
* '2' Symbolic link
|
||||
* '3' Character special
|
||||
* '4' Block special
|
||||
* '5' Directory
|
||||
* '6' FIFO
|
||||
* '7' Contigous
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
*
|
||||
* Ustar header
|
||||
*
|
||||
* <pre>
|
||||
* Offset Size Field
|
||||
* 257 6 UStar indicator "ustar"
|
||||
* 263 2 UStar version "00"
|
||||
* 265 32 Owner user name
|
||||
* 297 32 Owner group name
|
||||
* 329 8 Device major number
|
||||
* 337 8 Device minor number
|
||||
* 345 155 Filename prefix
|
||||
* </pre>
|
||||
*/
|
||||
|
||||
public class TarHeader {
|
||||
|
||||
/*
|
||||
* Header
|
||||
*/
|
||||
public static final int NAMELEN = 100;
|
||||
public static final int MODELEN = 8;
|
||||
public static final int UIDLEN = 8;
|
||||
public static final int GIDLEN = 8;
|
||||
public static final int SIZELEN = 12;
|
||||
public static final int MODTIMELEN = 12;
|
||||
public static final int CHKSUMLEN = 8;
|
||||
public static final byte LF_OLDNORM = 0;
|
||||
|
||||
/*
|
||||
* File Types
|
||||
*/
|
||||
public static final byte LF_NORMAL = (byte) '0';
|
||||
public static final byte LF_LINK = (byte) '1';
|
||||
public static final byte LF_SYMLINK = (byte) '2';
|
||||
public static final byte LF_CHR = (byte) '3';
|
||||
public static final byte LF_BLK = (byte) '4';
|
||||
public static final byte LF_DIR = (byte) '5';
|
||||
public static final byte LF_FIFO = (byte) '6';
|
||||
public static final byte LF_CONTIG = (byte) '7';
|
||||
|
||||
/*
|
||||
* Ustar header
|
||||
*/
|
||||
|
||||
public static final String USTAR_MAGIC = "ustar"; // POSIX
|
||||
|
||||
public static final int USTAR_MAGICLEN = 8;
|
||||
public static final int USTAR_USER_NAMELEN = 32;
|
||||
public static final int USTAR_GROUP_NAMELEN = 32;
|
||||
public static final int USTAR_DEVLEN = 8;
|
||||
public static final int USTAR_FILENAME_PREFIX = 155;
|
||||
|
||||
// Header values
|
||||
public StringBuffer name;
|
||||
public int mode;
|
||||
public int userId;
|
||||
public int groupId;
|
||||
public long size;
|
||||
public long modTime;
|
||||
public int checkSum;
|
||||
public byte linkFlag;
|
||||
public StringBuffer linkName;
|
||||
public StringBuffer magic; // ustar indicator and version
|
||||
public StringBuffer userName;
|
||||
public StringBuffer groupName;
|
||||
public int devMajor;
|
||||
public int devMinor;
|
||||
public StringBuffer namePrefix;
|
||||
|
||||
public TarHeader() {
|
||||
this.magic = new StringBuffer(TarHeader.USTAR_MAGIC);
|
||||
|
||||
this.name = new StringBuffer();
|
||||
this.linkName = new StringBuffer();
|
||||
|
||||
String user = System.getProperty("user.name", "");
|
||||
|
||||
if (user.length() > 31)
|
||||
user = user.substring(0, 31);
|
||||
|
||||
this.userId = 0;
|
||||
this.groupId = 0;
|
||||
this.userName = new StringBuffer(user);
|
||||
this.groupName = new StringBuffer("");
|
||||
this.namePrefix = new StringBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an entry name from a header buffer.
|
||||
*
|
||||
* @param name
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The header's entry name.
|
||||
*/
|
||||
public static StringBuffer parseName(byte[] header, int offset, int length) {
|
||||
StringBuffer result = new StringBuffer(length);
|
||||
|
||||
int end = offset + length;
|
||||
for (int i = offset; i < end; ++i) {
|
||||
if (header[i] == 0)
|
||||
break;
|
||||
result.append((char) header[i]);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the number of bytes in an entry name.
|
||||
*
|
||||
* @param name
|
||||
* @param header
|
||||
* The header buffer from which to parse.
|
||||
* @param offset
|
||||
* The offset into the buffer from which to parse.
|
||||
* @param length
|
||||
* The number of header bytes to parse.
|
||||
* @return The number of bytes in a header's entry name.
|
||||
*/
|
||||
public static int getNameBytes(StringBuffer name, byte[] buf, int offset, int length) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < length && i < name.length(); ++i) {
|
||||
buf[offset + i] = (byte) name.charAt(i);
|
||||
}
|
||||
|
||||
for (; i < length; ++i) {
|
||||
buf[offset + i] = 0;
|
||||
}
|
||||
|
||||
return offset + length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new header for a file/directory entry.
|
||||
*
|
||||
*
|
||||
* @param name
|
||||
* File name
|
||||
* @param size
|
||||
* File size in bytes
|
||||
* @param modTime
|
||||
* Last modification time in numeric Unix time format
|
||||
* @param dir
|
||||
* Is directory
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public static TarHeader createHeader(String entryName, long size, long modTime, boolean dir) {
|
||||
String name = entryName;
|
||||
name = TarUtils.trim(name.replace(File.separatorChar, '/'), '/');
|
||||
|
||||
TarHeader header = new TarHeader();
|
||||
header.linkName = new StringBuffer("");
|
||||
|
||||
if (name.length() > 100) {
|
||||
header.namePrefix = new StringBuffer(name.substring(0, name.lastIndexOf('/')));
|
||||
header.name = new StringBuffer(name.substring(name.lastIndexOf('/') + 1));
|
||||
} else {
|
||||
header.name = new StringBuffer(name);
|
||||
}
|
||||
|
||||
if (dir) {
|
||||
header.mode = 040755;
|
||||
header.linkFlag = TarHeader.LF_DIR;
|
||||
if (header.name.charAt(header.name.length() - 1) != '/') {
|
||||
header.name.append("/");
|
||||
}
|
||||
header.size = 0;
|
||||
} else {
|
||||
header.mode = 0100644;
|
||||
header.linkFlag = TarHeader.LF_NORMAL;
|
||||
header.size = size;
|
||||
}
|
||||
|
||||
header.modTime = modTime;
|
||||
header.checkSum = 0;
|
||||
header.devMajor = 0;
|
||||
header.devMinor = 0;
|
||||
|
||||
return header;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarInputStream extends FilterInputStream {
|
||||
|
||||
private static final int SKIP_BUFFER_SIZE = 2048;
|
||||
private TarEntry currentEntry;
|
||||
private long currentFileSize;
|
||||
private long bytesRead;
|
||||
private boolean defaultSkip = false;
|
||||
|
||||
public TarInputStream(InputStream in) {
|
||||
super(in);
|
||||
currentFileSize = 0;
|
||||
bytesRead = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean markSupported() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public synchronized void mark(int readlimit) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public synchronized void reset() throws IOException {
|
||||
throw new IOException("mark/reset not supported");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a byte
|
||||
*
|
||||
* @see java.io.FilterInputStream#read()
|
||||
*/
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
byte[] buf = new byte[1];
|
||||
|
||||
int res = this.read(buf, 0, 1);
|
||||
|
||||
if (res != -1) {
|
||||
return 0xFF & buf[0];
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the bytes being read exceed the entry size and adjusts the byte
|
||||
* array length. Updates the byte counters
|
||||
*
|
||||
*
|
||||
* @see java.io.FilterInputStream#read(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public int read(byte[] b, int off, int len) throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentFileSize == currentEntry.getSize()) {
|
||||
return -1;
|
||||
} else if ((currentEntry.getSize() - currentFileSize) < len) {
|
||||
len = (int) (currentEntry.getSize() - currentFileSize);
|
||||
}
|
||||
}
|
||||
|
||||
int br = super.read(b, off, len);
|
||||
|
||||
if (br != -1) {
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += br;
|
||||
}
|
||||
|
||||
bytesRead += br;
|
||||
}
|
||||
|
||||
return br;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the next entry in the tar file
|
||||
*
|
||||
* @return TarEntry
|
||||
* @throws IOException
|
||||
*/
|
||||
public TarEntry getNextEntry() throws IOException {
|
||||
closeCurrentEntry();
|
||||
|
||||
byte[] header = new byte[TarConstants.HEADER_BLOCK];
|
||||
byte[] theader = new byte[TarConstants.HEADER_BLOCK];
|
||||
int tr = 0;
|
||||
|
||||
// Read full header
|
||||
while (tr < TarConstants.HEADER_BLOCK) {
|
||||
int res = read(theader, 0, TarConstants.HEADER_BLOCK - tr);
|
||||
|
||||
if (res < 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
System.arraycopy(theader, 0, header, tr, res);
|
||||
tr += res;
|
||||
}
|
||||
|
||||
// Check if record is null
|
||||
boolean eof = true;
|
||||
for (byte b : header) {
|
||||
if (b != 0) {
|
||||
eof = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!eof) {
|
||||
currentEntry = new TarEntry(header);
|
||||
}
|
||||
|
||||
return currentEntry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the current offset (in bytes) from the beginning of the stream.
|
||||
* This can be used to find out at which point in a tar file an entry's content begins, for instance.
|
||||
*/
|
||||
public long getCurrentOffset() {
|
||||
return bytesRead;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the current tar entry
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void closeCurrentEntry() throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentEntry.getSize() > currentFileSize) {
|
||||
// Not fully read, skip rest of the bytes
|
||||
long bs = 0;
|
||||
while (bs < currentEntry.getSize() - currentFileSize) {
|
||||
long res = skip(currentEntry.getSize() - currentFileSize - bs);
|
||||
|
||||
if (res == 0 && currentEntry.getSize() - currentFileSize > 0) {
|
||||
// I suspect file corruption
|
||||
throw new IOException("Possible tar file corruption");
|
||||
}
|
||||
|
||||
bs += res;
|
||||
}
|
||||
}
|
||||
|
||||
currentEntry = null;
|
||||
currentFileSize = 0L;
|
||||
skipPad();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skips the pad at the end of each tar entry file content
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void skipPad() throws IOException {
|
||||
if (bytesRead > 0) {
|
||||
int extra = (int) (bytesRead % TarConstants.DATA_BLOCK);
|
||||
|
||||
if (extra > 0) {
|
||||
long bs = 0;
|
||||
while (bs < TarConstants.DATA_BLOCK - extra) {
|
||||
long res = skip(TarConstants.DATA_BLOCK - extra - bs);
|
||||
bs += res;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skips 'n' bytes on the InputStream<br>
|
||||
* Overrides default implementation of skip
|
||||
*
|
||||
*/
|
||||
@Override
|
||||
public long skip(long n) throws IOException {
|
||||
if (defaultSkip) {
|
||||
// use skip method of parent stream
|
||||
// may not work if skip not implemented by parent
|
||||
long bs = super.skip(n);
|
||||
bytesRead += bs;
|
||||
|
||||
return bs;
|
||||
}
|
||||
|
||||
if (n <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
long left = n;
|
||||
byte[] sBuff = new byte[SKIP_BUFFER_SIZE];
|
||||
|
||||
while (left > 0) {
|
||||
int res = read(sBuff, 0, (int) (left < SKIP_BUFFER_SIZE ? left : SKIP_BUFFER_SIZE));
|
||||
if (res < 0) {
|
||||
break;
|
||||
}
|
||||
left -= res;
|
||||
}
|
||||
|
||||
return n - left;
|
||||
}
|
||||
|
||||
public boolean isDefaultSkip() {
|
||||
return defaultSkip;
|
||||
}
|
||||
|
||||
public void setDefaultSkip(boolean defaultSkip) {
|
||||
this.defaultSkip = defaultSkip;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
|
||||
/**
|
||||
* @author Kamran Zafar
|
||||
*
|
||||
*/
|
||||
public class TarOutputStream extends OutputStream {
|
||||
private final OutputStream out;
|
||||
private long bytesWritten;
|
||||
private long currentFileSize;
|
||||
private TarEntry currentEntry;
|
||||
|
||||
public TarOutputStream(OutputStream out) {
|
||||
this.out = out;
|
||||
bytesWritten = 0;
|
||||
currentFileSize = 0;
|
||||
}
|
||||
|
||||
public TarOutputStream(final File fout) throws FileNotFoundException {
|
||||
this.out = new BufferedOutputStream(new FileOutputStream(fout));
|
||||
bytesWritten = 0;
|
||||
currentFileSize = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a file for writing.
|
||||
*/
|
||||
public TarOutputStream(final File fout, final boolean append) throws IOException {
|
||||
@SuppressWarnings("resource")
|
||||
RandomAccessFile raf = new RandomAccessFile(fout, "rw");
|
||||
final long fileSize = fout.length();
|
||||
if (append && fileSize > TarConstants.EOF_BLOCK) {
|
||||
raf.seek(fileSize - TarConstants.EOF_BLOCK);
|
||||
}
|
||||
out = new BufferedOutputStream(new FileOutputStream(raf.getFD()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends the EOF record and closes the stream
|
||||
*
|
||||
* @see java.io.FilterOutputStream#close()
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
closeCurrentEntry();
|
||||
write( new byte[TarConstants.EOF_BLOCK] );
|
||||
out.close();
|
||||
}
|
||||
/**
|
||||
* Writes a byte to the stream and updates byte counters
|
||||
*
|
||||
* @see java.io.FilterOutputStream#write(int)
|
||||
*/
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
out.write( b );
|
||||
bytesWritten += 1;
|
||||
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the bytes being written exceed the current entry size.
|
||||
*
|
||||
* @see java.io.FilterOutputStream#write(byte[], int, int)
|
||||
*/
|
||||
@Override
|
||||
public void write(byte[] b, int off, int len) throws IOException {
|
||||
if (currentEntry != null && !currentEntry.isDirectory()) {
|
||||
if (currentEntry.getSize() < currentFileSize + len) {
|
||||
throw new IOException( "The current entry[" + currentEntry.getName() + "] size["
|
||||
+ currentEntry.getSize() + "] is smaller than the bytes[" + ( currentFileSize + len )
|
||||
+ "] being written." );
|
||||
}
|
||||
}
|
||||
|
||||
out.write( b, off, len );
|
||||
|
||||
bytesWritten += len;
|
||||
|
||||
if (currentEntry != null) {
|
||||
currentFileSize += len;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes the next tar entry header on the stream
|
||||
*
|
||||
* @param entry
|
||||
* @throws IOException
|
||||
*/
|
||||
public void putNextEntry(TarEntry entry) throws IOException {
|
||||
closeCurrentEntry();
|
||||
|
||||
byte[] header = new byte[TarConstants.HEADER_BLOCK];
|
||||
entry.writeEntryHeader( header );
|
||||
|
||||
write( header );
|
||||
|
||||
currentEntry = entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the current tar entry
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void closeCurrentEntry() throws IOException {
|
||||
if (currentEntry != null) {
|
||||
if (currentEntry.getSize() > currentFileSize) {
|
||||
throw new IOException( "The current entry[" + currentEntry.getName() + "] of size["
|
||||
+ currentEntry.getSize() + "] has not been fully written." );
|
||||
}
|
||||
|
||||
currentEntry = null;
|
||||
currentFileSize = 0;
|
||||
|
||||
pad();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pads the last content block
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void pad() throws IOException {
|
||||
if (bytesWritten > 0) {
|
||||
int extra = (int) ( bytesWritten % TarConstants.DATA_BLOCK );
|
||||
|
||||
if (extra > 0) {
|
||||
write( new byte[TarConstants.DATA_BLOCK - extra] );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Copyright 2012 Kamran Zafar
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package org.kamranzafar.jtar;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* @author Kamran
|
||||
*
|
||||
*/
|
||||
public class TarUtils {
|
||||
/**
|
||||
* Determines the tar file size of the given folder/file path
|
||||
*
|
||||
* @param path
|
||||
* @return
|
||||
*/
|
||||
public static long calculateTarSize(File path) {
|
||||
return tarSize(path) + TarConstants.EOF_BLOCK;
|
||||
}
|
||||
|
||||
private static long tarSize(File dir) {
|
||||
long size = 0;
|
||||
|
||||
if (dir.isFile()) {
|
||||
return entrySize(dir.length());
|
||||
} else {
|
||||
File[] subFiles = dir.listFiles();
|
||||
|
||||
if (subFiles != null && subFiles.length > 0) {
|
||||
for (File file : subFiles) {
|
||||
if (file.isFile()) {
|
||||
size += entrySize(file.length());
|
||||
} else {
|
||||
size += tarSize(file);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Empty folder header
|
||||
return TarConstants.HEADER_BLOCK;
|
||||
}
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
private static long entrySize(long fileSize) {
|
||||
long size = 0;
|
||||
size += TarConstants.HEADER_BLOCK; // Header
|
||||
size += fileSize; // File size
|
||||
|
||||
long extra = size % TarConstants.DATA_BLOCK;
|
||||
|
||||
if (extra > 0) {
|
||||
size += (TarConstants.DATA_BLOCK - extra); // pad
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
public static String trim(String s, char c) {
|
||||
StringBuffer tmp = new StringBuffer(s);
|
||||
for (int i = 0; i < tmp.length(); i++) {
|
||||
if (tmp.charAt(i) != c) {
|
||||
break;
|
||||
} else {
|
||||
tmp.deleteCharAt(i);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = tmp.length() - 1; i >= 0; i--) {
|
||||
if (tmp.charAt(i) != c) {
|
||||
break;
|
||||
} else {
|
||||
tmp.deleteCharAt(i);
|
||||
}
|
||||
}
|
||||
|
||||
return tmp.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
|
||||
package org.kivy.android;
|
||||
|
||||
import java.net.Socket;
|
||||
import java.net.InetSocketAddress;
|
||||
|
||||
import android.os.SystemClock;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import android.app.*;
|
||||
import android.content.*;
|
||||
import android.view.*;
|
||||
import android.view.SurfaceView;
|
||||
import android.app.Activity;
|
||||
import android.content.Intent;
|
||||
import android.util.Log;
|
||||
import android.widget.Toast;
|
||||
import android.os.AsyncTask;
|
||||
import android.os.Bundle;
|
||||
import android.os.PowerManager;
|
||||
import android.graphics.PixelFormat;
|
||||
import android.view.SurfaceHolder;
|
||||
import android.content.Context;
|
||||
import android.content.pm.ActivityInfo;
|
||||
import android.content.pm.PackageManager;
|
||||
import android.content.pm.ApplicationInfo;
|
||||
import android.content.Intent;
|
||||
import android.widget.ImageView;
|
||||
import java.io.InputStream;
|
||||
import android.graphics.Bitmap;
|
||||
import android.graphics.BitmapFactory;
|
||||
import android.graphics.Color;
|
||||
|
||||
import android.widget.AbsoluteLayout;
|
||||
|
||||
import android.webkit.WebViewClient;
|
||||
import android.webkit.WebView;
|
||||
|
||||
import org.kivy.android.PythonUtil;
|
||||
|
||||
import org.renpy.android.ResourceManager;
|
||||
import org.renpy.android.AssetExtract;
|
||||
|
||||
public class PythonActivity extends Activity {
|
||||
// This activity is modified from a mixture of the SDLActivity and
|
||||
// PythonActivity in the SDL2 bootstrap, but removing all the SDL2
|
||||
// specifics.
|
||||
|
||||
private static final String TAG = "PythonActivity";
|
||||
|
||||
public static PythonActivity mActivity = null;
|
||||
|
||||
/** If shared libraries (e.g. the native application) could not be loaded. */
|
||||
public static boolean mBrokenLibraries;
|
||||
|
||||
protected static Thread mPythonThread;
|
||||
|
||||
private ResourceManager resourceManager = null;
|
||||
private Bundle mMetaData = null;
|
||||
private PowerManager.WakeLock mWakeLock = null;
|
||||
|
||||
public String getAppRoot() {
|
||||
String app_root = getFilesDir().getAbsolutePath() + "/app";
|
||||
return app_root;
|
||||
}
|
||||
|
||||
public static void initialize() {
|
||||
// The static nature of the singleton and Android quirkiness force us to initialize everything here
|
||||
// Otherwise, when exiting the app and returning to it, these variables *keep* their pre exit values
|
||||
mBrokenLibraries = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onCreate(Bundle savedInstanceState) {
|
||||
Log.v(TAG, "My oncreate running");
|
||||
resourceManager = new ResourceManager(this);
|
||||
|
||||
Log.v(TAG, "Ready to unpack");
|
||||
File app_root_file = new File(getAppRoot());
|
||||
unpackData("private", app_root_file);
|
||||
|
||||
Log.v(TAG, "About to do super onCreate");
|
||||
super.onCreate(savedInstanceState);
|
||||
Log.v(TAG, "Did super onCreate");
|
||||
|
||||
this.mActivity = this;
|
||||
//this.showLoadingScreen();
|
||||
Log.v("Python", "Device: " + android.os.Build.DEVICE);
|
||||
Log.v("Python", "Model: " + android.os.Build.MODEL);
|
||||
|
||||
//Log.v(TAG, "Ready to unpack");
|
||||
//new UnpackFilesTask().execute(getAppRoot());
|
||||
|
||||
PythonActivity.initialize();
|
||||
|
||||
// Load shared libraries
|
||||
String errorMsgBrokenLib = "";
|
||||
try {
|
||||
loadLibraries();
|
||||
} catch(UnsatisfiedLinkError e) {
|
||||
System.err.println(e.getMessage());
|
||||
mBrokenLibraries = true;
|
||||
errorMsgBrokenLib = e.getMessage();
|
||||
} catch(Exception e) {
|
||||
System.err.println(e.getMessage());
|
||||
mBrokenLibraries = true;
|
||||
errorMsgBrokenLib = e.getMessage();
|
||||
}
|
||||
|
||||
if (mBrokenLibraries)
|
||||
{
|
||||
AlertDialog.Builder dlgAlert = new AlertDialog.Builder(this);
|
||||
dlgAlert.setMessage("An error occurred while trying to load the application libraries. Please try again and/or reinstall."
|
||||
+ System.getProperty("line.separator")
|
||||
+ System.getProperty("line.separator")
|
||||
+ "Error: " + errorMsgBrokenLib);
|
||||
dlgAlert.setTitle("Python Error");
|
||||
dlgAlert.setPositiveButton("Exit",
|
||||
new DialogInterface.OnClickListener() {
|
||||
@Override
|
||||
public void onClick(DialogInterface dialog,int id) {
|
||||
// if this button is clicked, close current activity
|
||||
PythonActivity.mActivity.finish();
|
||||
}
|
||||
});
|
||||
dlgAlert.setCancelable(false);
|
||||
dlgAlert.create().show();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Set up the Python environment
|
||||
String app_root_dir = getAppRoot();
|
||||
String mFilesDirectory = mActivity.getFilesDir().getAbsolutePath();
|
||||
|
||||
Log.v(TAG, "Setting env vars for start.c and Python to use");
|
||||
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
|
||||
PythonActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
|
||||
PythonActivity.nativeSetEnv("ANDROID_UNPACK", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("PYTHONHOME", app_root_dir);
|
||||
PythonActivity.nativeSetEnv("PYTHONPATH", app_root_dir + ":" + app_root_dir + "/lib");
|
||||
PythonActivity.nativeSetEnv("PYTHONOPTIMIZE", "2");
|
||||
|
||||
try {
|
||||
Log.v(TAG, "Access to our meta-data...");
|
||||
mActivity.mMetaData = mActivity.getPackageManager().getApplicationInfo(
|
||||
mActivity.getPackageName(), PackageManager.GET_META_DATA).metaData;
|
||||
|
||||
PowerManager pm = (PowerManager) mActivity.getSystemService(Context.POWER_SERVICE);
|
||||
if ( mActivity.mMetaData.getInt("wakelock") == 1 ) {
|
||||
mActivity.mWakeLock = pm.newWakeLock(PowerManager.SCREEN_BRIGHT_WAKE_LOCK, "Screen On");
|
||||
mActivity.mWakeLock.acquire();
|
||||
}
|
||||
} catch (PackageManager.NameNotFoundException e) {
|
||||
}
|
||||
|
||||
final Thread pythonThread = new Thread(new PythonMain(), "PythonThread");
|
||||
PythonActivity.mPythonThread = pythonThread;
|
||||
pythonThread.start();
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onDestroy() {
|
||||
Log.i("Destroy", "end of app");
|
||||
super.onDestroy();
|
||||
|
||||
// make sure all child threads (python_thread) are stopped
|
||||
android.os.Process.killProcess(android.os.Process.myPid());
|
||||
}
|
||||
|
||||
public void loadLibraries() {
|
||||
String app_root = new String(getAppRoot());
|
||||
File app_root_file = new File(app_root);
|
||||
PythonUtil.loadLibraries(app_root_file);
|
||||
}
|
||||
|
||||
public void recursiveDelete(File f) {
|
||||
if (f.isDirectory()) {
|
||||
for (File r : f.listFiles()) {
|
||||
recursiveDelete(r);
|
||||
}
|
||||
}
|
||||
f.delete();
|
||||
}
|
||||
|
||||
/**
|
||||
* Show an error using a toast. (Only makes sense from non-UI
|
||||
* threads.)
|
||||
*/
|
||||
public void toastError(final String msg) {
|
||||
|
||||
final Activity thisActivity = this;
|
||||
|
||||
runOnUiThread(new Runnable () {
|
||||
public void run() {
|
||||
Toast.makeText(thisActivity, msg, Toast.LENGTH_LONG).show();
|
||||
}
|
||||
});
|
||||
|
||||
// Wait to show the error.
|
||||
synchronized (this) {
|
||||
try {
|
||||
this.wait(1000);
|
||||
} catch (InterruptedException e) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void unpackData(final String resource, File target) {
|
||||
|
||||
Log.v(TAG, "UNPACKING!!! " + resource + " " + target.getName());
|
||||
|
||||
// The version of data in memory and on disk.
|
||||
String data_version = resourceManager.getString(resource + "_version");
|
||||
String disk_version = null;
|
||||
|
||||
Log.v(TAG, "Data version is " + data_version);
|
||||
|
||||
// If no version, no unpacking is necessary.
|
||||
if (data_version == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check the current disk version, if any.
|
||||
String filesDir = target.getAbsolutePath();
|
||||
String disk_version_fn = filesDir + "/" + resource + ".version";
|
||||
|
||||
try {
|
||||
byte buf[] = new byte[64];
|
||||
InputStream is = new FileInputStream(disk_version_fn);
|
||||
int len = is.read(buf);
|
||||
disk_version = new String(buf, 0, len);
|
||||
is.close();
|
||||
} catch (Exception e) {
|
||||
disk_version = "";
|
||||
}
|
||||
|
||||
// If the disk data is out of date, extract it and write the
|
||||
// version file.
|
||||
// if (! data_version.equals(disk_version)) {
|
||||
if (! data_version.equals(disk_version)) {
|
||||
Log.v(TAG, "Extracting " + resource + " assets.");
|
||||
|
||||
recursiveDelete(target);
|
||||
target.mkdirs();
|
||||
|
||||
AssetExtract ae = new AssetExtract(this);
|
||||
if (!ae.extractTar(resource + ".mp3", target.getAbsolutePath())) {
|
||||
toastError("Could not extract " + resource + " data.");
|
||||
}
|
||||
|
||||
try {
|
||||
// Write .nomedia.
|
||||
new File(target, ".nomedia").createNewFile();
|
||||
|
||||
// Write version file.
|
||||
FileOutputStream os = new FileOutputStream(disk_version_fn);
|
||||
os.write(data_version.getBytes());
|
||||
os.close();
|
||||
} catch (Exception e) {
|
||||
Log.w("python", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long lastBackClick = SystemClock.elapsedRealtime();
|
||||
@Override
|
||||
public boolean onKeyDown(int keyCode, KeyEvent event) {
|
||||
// If it wasn't the Back key or there's no web page history, bubble up to the default
|
||||
// system behavior (probably exit the activity)
|
||||
if (SystemClock.elapsedRealtime() - lastBackClick > 2000){
|
||||
lastBackClick = SystemClock.elapsedRealtime();
|
||||
Toast.makeText(this, "Click again to close the app",
|
||||
Toast.LENGTH_LONG).show();
|
||||
return true;
|
||||
}
|
||||
|
||||
lastBackClick = SystemClock.elapsedRealtime();
|
||||
return super.onKeyDown(keyCode, event);
|
||||
}
|
||||
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
// Listener interface for onNewIntent
|
||||
//
|
||||
|
||||
public interface NewIntentListener {
|
||||
void onNewIntent(Intent intent);
|
||||
}
|
||||
|
||||
private List<NewIntentListener> newIntentListeners = null;
|
||||
|
||||
public void registerNewIntentListener(NewIntentListener listener) {
|
||||
if ( this.newIntentListeners == null )
|
||||
this.newIntentListeners = Collections.synchronizedList(new ArrayList<NewIntentListener>());
|
||||
this.newIntentListeners.add(listener);
|
||||
}
|
||||
|
||||
public void unregisterNewIntentListener(NewIntentListener listener) {
|
||||
if ( this.newIntentListeners == null )
|
||||
return;
|
||||
this.newIntentListeners.remove(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onNewIntent(Intent intent) {
|
||||
if ( this.newIntentListeners == null )
|
||||
return;
|
||||
this.onResume();
|
||||
synchronized ( this.newIntentListeners ) {
|
||||
Iterator<NewIntentListener> iterator = this.newIntentListeners.iterator();
|
||||
while ( iterator.hasNext() ) {
|
||||
(iterator.next()).onNewIntent(intent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
// Listener interface for onActivityResult
|
||||
//
|
||||
|
||||
public interface ActivityResultListener {
|
||||
void onActivityResult(int requestCode, int resultCode, Intent data);
|
||||
}
|
||||
|
||||
private List<ActivityResultListener> activityResultListeners = null;
|
||||
|
||||
public void registerActivityResultListener(ActivityResultListener listener) {
|
||||
if ( this.activityResultListeners == null )
|
||||
this.activityResultListeners = Collections.synchronizedList(new ArrayList<ActivityResultListener>());
|
||||
this.activityResultListeners.add(listener);
|
||||
}
|
||||
|
||||
public void unregisterActivityResultListener(ActivityResultListener listener) {
|
||||
if ( this.activityResultListeners == null )
|
||||
return;
|
||||
this.activityResultListeners.remove(listener);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onActivityResult(int requestCode, int resultCode, Intent intent) {
|
||||
if ( this.activityResultListeners == null )
|
||||
return;
|
||||
this.onResume();
|
||||
synchronized ( this.activityResultListeners ) {
|
||||
Iterator<ActivityResultListener> iterator = this.activityResultListeners.iterator();
|
||||
while ( iterator.hasNext() )
|
||||
(iterator.next()).onActivityResult(requestCode, resultCode, intent);
|
||||
}
|
||||
}
|
||||
|
||||
public static void start_service(String serviceTitle, String serviceDescription,
|
||||
String pythonServiceArgument) {
|
||||
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
|
||||
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
|
||||
String filesDirectory = argument;
|
||||
String app_root_dir = PythonActivity.mActivity.getAppRoot();
|
||||
serviceIntent.putExtra("androidPrivate", argument);
|
||||
serviceIntent.putExtra("androidArgument", app_root_dir);
|
||||
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
|
||||
serviceIntent.putExtra("pythonName", "python");
|
||||
serviceIntent.putExtra("pythonHome", app_root_dir);
|
||||
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
|
||||
serviceIntent.putExtra("serviceTitle", serviceTitle);
|
||||
serviceIntent.putExtra("serviceDescription", serviceDescription);
|
||||
serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument);
|
||||
PythonActivity.mActivity.startService(serviceIntent);
|
||||
}
|
||||
|
||||
public static void stop_service() {
|
||||
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
|
||||
PythonActivity.mActivity.stopService(serviceIntent);
|
||||
}
|
||||
|
||||
|
||||
public static native void nativeSetEnv(String j_name, String j_value);
|
||||
public static native int nativeInit(Object arguments);
|
||||
|
||||
}
|
||||
|
||||
|
||||
class PythonMain implements Runnable {
|
||||
@Override
|
||||
public void run() {
|
||||
PythonActivity.nativeInit(new String[0]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package org.kivy.android.concurrency;
|
||||
|
||||
import java.util.concurrent.locks.Condition;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Created by ryan on 3/28/14.
|
||||
*/
|
||||
public class PythonEvent {
|
||||
private final Lock lock = new ReentrantLock();
|
||||
private final Condition cond = lock.newCondition();
|
||||
private boolean flag = false;
|
||||
|
||||
public void set() {
|
||||
lock.lock();
|
||||
try {
|
||||
flag = true;
|
||||
cond.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void wait_() throws InterruptedException {
|
||||
lock.lock();
|
||||
try {
|
||||
while (!flag) {
|
||||
cond.await();
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
lock.lock();
|
||||
try {
|
||||
flag = false;
|
||||
cond.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package org.kivy.android.concurrency;
|
||||
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Created by ryan on 3/28/14.
|
||||
*/
|
||||
public class PythonLock {
|
||||
private final Lock lock = new ReentrantLock();
|
||||
|
||||
public void acquire() {
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package org.renpy.android;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import android.content.BroadcastReceiver;
|
||||
import android.content.Context;
|
||||
import android.content.Intent;
|
||||
import android.content.IntentFilter;
|
||||
import android.hardware.Sensor;
|
||||
import android.hardware.SensorEvent;
|
||||
import android.hardware.SensorEventListener;
|
||||
import android.hardware.SensorManager;
|
||||
import android.net.ConnectivityManager;
|
||||
import android.net.NetworkInfo;
|
||||
import android.net.wifi.ScanResult;
|
||||
import android.net.wifi.WifiManager;
|
||||
import android.os.Vibrator;
|
||||
import android.view.View;
|
||||
|
||||
/**
|
||||
* Methods that are expected to be called via JNI, to access the device's
|
||||
* non-screen hardware. (For example, the vibration and accelerometer.)
|
||||
*/
|
||||
public class Hardware {
|
||||
|
||||
// The context.
|
||||
static Context context;
|
||||
static View view;
|
||||
|
||||
/**
|
||||
* Vibrate for s seconds.
|
||||
*/
|
||||
public static void vibrate(double s) {
|
||||
Vibrator v = (Vibrator) context
|
||||
.getSystemService(Context.VIBRATOR_SERVICE);
|
||||
if (v != null) {
|
||||
v.vibrate((int) (1000 * s));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an Overview of all Hardware Sensors of an Android Device
|
||||
*/
|
||||
public static String getHardwareSensors() {
|
||||
SensorManager sm = (SensorManager) context
|
||||
.getSystemService(Context.SENSOR_SERVICE);
|
||||
List<Sensor> allSensors = sm.getSensorList(Sensor.TYPE_ALL);
|
||||
|
||||
if (allSensors != null) {
|
||||
String resultString = "";
|
||||
for (Sensor s : allSensors) {
|
||||
resultString += String.format("Name=" + s.getName());
|
||||
resultString += String.format(",Vendor=" + s.getVendor());
|
||||
resultString += String.format(",Version=" + s.getVersion());
|
||||
resultString += String.format(",MaximumRange="
|
||||
+ s.getMaximumRange());
|
||||
// XXX MinDelay is not in the 2.2
|
||||
// resultString += String.format(",MinDelay=" +
|
||||
// s.getMinDelay());
|
||||
resultString += String.format(",Power=" + s.getPower());
|
||||
resultString += String.format(",Type=" + s.getType() + "\n");
|
||||
}
|
||||
return resultString;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Access to 3 Axis Hardware Sensors Accelerometer, Orientation and
|
||||
* Magnetic Field Sensors
|
||||
*/
|
||||
public static class generic3AxisSensor implements SensorEventListener {
|
||||
private final SensorManager sSensorManager;
|
||||
private final Sensor sSensor;
|
||||
private final int sSensorType;
|
||||
SensorEvent sSensorEvent;
|
||||
|
||||
public generic3AxisSensor(int sensorType) {
|
||||
sSensorType = sensorType;
|
||||
sSensorManager = (SensorManager) context
|
||||
.getSystemService(Context.SENSOR_SERVICE);
|
||||
sSensor = sSensorManager.getDefaultSensor(sSensorType);
|
||||
}
|
||||
|
||||
public void onAccuracyChanged(Sensor sensor, int accuracy) {
|
||||
}
|
||||
|
||||
public void onSensorChanged(SensorEvent event) {
|
||||
sSensorEvent = event;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable or disable the Sensor by registering/unregistering
|
||||
*/
|
||||
public void changeStatus(boolean enable) {
|
||||
if (enable) {
|
||||
sSensorManager.registerListener(this, sSensor,
|
||||
SensorManager.SENSOR_DELAY_NORMAL);
|
||||
} else {
|
||||
sSensorManager.unregisterListener(this, sSensor);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the Sensor
|
||||
*/
|
||||
public float[] readSensor() {
|
||||
if (sSensorEvent != null) {
|
||||
return sSensorEvent.values;
|
||||
} else {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static generic3AxisSensor accelerometerSensor = null;
|
||||
public static generic3AxisSensor orientationSensor = null;
|
||||
public static generic3AxisSensor magneticFieldSensor = null;
|
||||
|
||||
/**
|
||||
* functions for backward compatibility reasons
|
||||
*/
|
||||
|
||||
public static void accelerometerEnable(boolean enable) {
|
||||
if (accelerometerSensor == null)
|
||||
accelerometerSensor = new generic3AxisSensor(
|
||||
Sensor.TYPE_ACCELEROMETER);
|
||||
accelerometerSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] accelerometerReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (accelerometerSensor == null)
|
||||
return rv;
|
||||
return (float[]) accelerometerSensor.readSensor();
|
||||
}
|
||||
|
||||
public static void orientationSensorEnable(boolean enable) {
|
||||
if (orientationSensor == null)
|
||||
orientationSensor = new generic3AxisSensor(Sensor.TYPE_ORIENTATION);
|
||||
orientationSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] orientationSensorReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (orientationSensor == null)
|
||||
return rv;
|
||||
return (float[]) orientationSensor.readSensor();
|
||||
}
|
||||
|
||||
public static void magneticFieldSensorEnable(boolean enable) {
|
||||
if (magneticFieldSensor == null)
|
||||
magneticFieldSensor = new generic3AxisSensor(
|
||||
Sensor.TYPE_MAGNETIC_FIELD);
|
||||
magneticFieldSensor.changeStatus(enable);
|
||||
}
|
||||
|
||||
public static float[] magneticFieldSensorReading() {
|
||||
float rv[] = { 0f, 0f, 0f };
|
||||
if (magneticFieldSensor == null)
|
||||
return rv;
|
||||
return (float[]) magneticFieldSensor.readSensor();
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan WiFi networks
|
||||
*/
|
||||
static List<ScanResult> latestResult;
|
||||
|
||||
public static void enableWifiScanner() {
|
||||
IntentFilter i = new IntentFilter();
|
||||
i.addAction(WifiManager.SCAN_RESULTS_AVAILABLE_ACTION);
|
||||
|
||||
context.registerReceiver(new BroadcastReceiver() {
|
||||
|
||||
@Override
|
||||
public void onReceive(Context c, Intent i) {
|
||||
// Code to execute when SCAN_RESULTS_AVAILABLE_ACTION event
|
||||
// occurs
|
||||
WifiManager w = (WifiManager) c
|
||||
.getSystemService(Context.WIFI_SERVICE);
|
||||
latestResult = w.getScanResults(); // Returns a <list> of
|
||||
// scanResults
|
||||
}
|
||||
|
||||
}, i);
|
||||
|
||||
}
|
||||
|
||||
public static String scanWifi() {
|
||||
|
||||
// Now you can call this and it should execute the broadcastReceiver's
|
||||
// onReceive()
|
||||
WifiManager wm = (WifiManager) context
|
||||
.getSystemService(Context.WIFI_SERVICE);
|
||||
boolean a = wm.startScan();
|
||||
|
||||
if (latestResult != null) {
|
||||
|
||||
String latestResultString = "";
|
||||
for (ScanResult result : latestResult) {
|
||||
latestResultString += String.format("%s\t%s\t%d\n",
|
||||
result.SSID, result.BSSID, result.level);
|
||||
}
|
||||
|
||||
return latestResultString;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* network state
|
||||
*/
|
||||
|
||||
public static boolean network_state = false;
|
||||
|
||||
/**
|
||||
* Check network state directly
|
||||
*
|
||||
* (only one connection can be active at a given moment, detects all network
|
||||
* type)
|
||||
*
|
||||
*/
|
||||
public static boolean checkNetwork() {
|
||||
boolean state = false;
|
||||
final ConnectivityManager conMgr = (ConnectivityManager) context
|
||||
.getSystemService(Context.CONNECTIVITY_SERVICE);
|
||||
|
||||
final NetworkInfo activeNetwork = conMgr.getActiveNetworkInfo();
|
||||
if (activeNetwork != null && activeNetwork.isConnected()) {
|
||||
state = true;
|
||||
} else {
|
||||
state = false;
|
||||
}
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* To recieve network state changes
|
||||
*/
|
||||
public static void registerNetworkCheck() {
|
||||
IntentFilter i = new IntentFilter();
|
||||
i.addAction(ConnectivityManager.CONNECTIVITY_ACTION);
|
||||
context.registerReceiver(new BroadcastReceiver() {
|
||||
|
||||
@Override
|
||||
public void onReceive(Context c, Intent i) {
|
||||
network_state = checkNetwork();
|
||||
}
|
||||
|
||||
}, i);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="{{ args.package }}"
|
||||
android:versionCode="{{ args.numeric_version }}"
|
||||
android:versionName="{{ args.version }}"
|
||||
android:installLocation="auto">
|
||||
|
||||
<supports-screens
|
||||
android:smallScreens="true"
|
||||
android:normalScreens="true"
|
||||
android:largeScreens="true"
|
||||
android:anyDensity="true"
|
||||
{% if args.min_sdk_version >= 9 %}
|
||||
android:xlargeScreens="true"
|
||||
{% endif %}
|
||||
/>
|
||||
|
||||
<!-- Android 2.3.3 -->
|
||||
<uses-sdk android:minSdkVersion="{{ args.min_sdk_version }}" android:targetSdkVersion="{{ android_api }}" />
|
||||
|
||||
<!-- Set permissions -->
|
||||
{% for perm in args.permissions %}
|
||||
{% if '.' in perm %}
|
||||
<uses-permission android:name="{{ perm }}" />
|
||||
{% else %}
|
||||
<uses-permission android:name="android.permission.{{ perm }}" />
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
{% if args.wakelock %}
|
||||
<uses-permission android:name="android.permission.WAKE_LOCK" />
|
||||
{% endif %}
|
||||
|
||||
{% if args.billing_pubkey %}
|
||||
<uses-permission android:name="com.android.vending.BILLING" />
|
||||
{% endif %}
|
||||
|
||||
<!-- Create a Java class extending SDLActivity and place it in a
|
||||
directory under src matching the package, e.g.
|
||||
src/com/gamemaker/game/MyGame.java
|
||||
|
||||
then replace "SDLActivity" with the name of your class (e.g. "MyGame")
|
||||
in the XML below.
|
||||
|
||||
An example Java class can be found in README-android.txt
|
||||
-->
|
||||
<application android:label="@string/app_name"
|
||||
android:icon="@drawable/icon"
|
||||
android:allowBackup="true"
|
||||
android:theme="@android:style/Theme.NoTitleBar{% if not args.window %}.Fullscreen{% endif %}"
|
||||
android:hardwareAccelerated="true" >
|
||||
|
||||
{% for m in args.meta_data %}
|
||||
<meta-data android:name="{{ m.split('=', 1)[0] }}" android:value="{{ m.split('=', 1)[-1] }}"/>{% endfor %}
|
||||
<meta-data android:name="wakelock" android:value="{% if args.wakelock %}1{% else %}0{% endif %}"/>
|
||||
|
||||
<activity android:name="org.kivy.android.PythonActivity"
|
||||
android:label="@string/app_name"
|
||||
android:configChanges="keyboardHidden|orientation{% if args.min_sdk_version >= 13 %}|screenSize{% endif %}"
|
||||
>
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
{%- if args.intent_filters -%}
|
||||
{{- args.intent_filters -}}
|
||||
{%- endif -%}
|
||||
</activity>
|
||||
|
||||
{% if service %}
|
||||
<service android:name="org.kivy.android.PythonService"
|
||||
android:process=":pythonservice" />
|
||||
{% endif %}
|
||||
{% for name in service_names %}
|
||||
<service android:name="{{ args.package }}.Service{{ name|capitalize }}"
|
||||
android:process=":service_{{ name }}" />
|
||||
{% endfor %}
|
||||
|
||||
{% if args.billing_pubkey %}
|
||||
<service android:name="org.kivy.android.billing.BillingReceiver"
|
||||
android:process=":pythonbilling" />
|
||||
<receiver android:name="org.kivy.android.billing.BillingReceiver"
|
||||
android:process=":pythonbillingreceiver">
|
||||
<intent-filter>
|
||||
<action android:name="com.android.vending.billing.IN_APP_NOTIFY" />
|
||||
<action android:name="com.android.vending.billing.RESPONSE_CODE" />
|
||||
<action android:name="com.android.vending.billing.PURCHASE_STATE_CHANGED" />
|
||||
</intent-filter>
|
||||
</receiver>
|
||||
{% endif %}
|
||||
</application>
|
||||
|
||||
</manifest>
|
||||
@@ -0,0 +1,74 @@
|
||||
package {{ args.package }};
|
||||
|
||||
import android.os.Binder;
|
||||
import android.os.IBinder;
|
||||
import android.content.Intent;
|
||||
import android.content.Context;
|
||||
import org.kivy.android.PythonService;
|
||||
|
||||
public class Service{{ name|capitalize }} extends PythonService {
|
||||
/**
|
||||
* Binder given to clients
|
||||
*/
|
||||
private final IBinder mBinder = new Service{{ name|capitalize }}Binder();
|
||||
|
||||
{% if sticky %}
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public int getStartType() {
|
||||
return START_STICKY;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{% if foreground %}
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public boolean getStartForeground() {
|
||||
return true;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
public static void start(Context ctx, String pythonServiceArgument) {
|
||||
String argument = ctx.getFilesDir().getAbsolutePath() + "/app";
|
||||
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
|
||||
intent.putExtra("androidPrivate", argument);
|
||||
intent.putExtra("androidArgument", argument);
|
||||
intent.putExtra("serviceEntrypoint", "{{ entrypoint }}");
|
||||
intent.putExtra("serviceTitle", "{{ name|capitalize }}");
|
||||
intent.putExtra("serviceDescription", "");
|
||||
intent.putExtra("pythonName", "{{ name }}");
|
||||
intent.putExtra("pythonHome", argument);
|
||||
intent.putExtra("androidUnpack", argument);
|
||||
intent.putExtra("pythonPath", argument + ":" + argument + "/lib");
|
||||
intent.putExtra("pythonServiceArgument", pythonServiceArgument);
|
||||
ctx.startService(intent);
|
||||
}
|
||||
|
||||
public static void stop(Context ctx) {
|
||||
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
|
||||
ctx.stopService(intent);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public IBinder onBind(Intent intent) {
|
||||
return mBinder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Class used for the client Binder. Because we know this service always
|
||||
* runs in the same process as its clients, we don't need to deal with IPC.
|
||||
*/
|
||||
public class Service{{ name|capitalize }}Binder extends Binder {
|
||||
Service{{ name|capitalize }} getService() {
|
||||
// Return this instance of Service{{ name|capitalize }} so clients can call public methods
|
||||
return Service{{ name|capitalize }}.this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<resources>
|
||||
<string name="app_name">{{ args.name }}</string>
|
||||
<string name="private_version">{{ private_version }}</string>
|
||||
</resources>
|
||||
@@ -0,0 +1,68 @@
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
from jnius import PythonJavaClass, autoclass, java_method
|
||||
JAVA_NAMESPACE = 'org.kivy.android'
|
||||
JNI_NAMESPACE = 'org/kivy/android'
|
||||
|
||||
_activity = autoclass(JAVA_NAMESPACE + '.PythonActivity').mActivity
|
||||
|
||||
_callbacks = {
|
||||
'on_new_intent': [],
|
||||
'on_activity_result': [],
|
||||
}
|
||||
|
||||
|
||||
class NewIntentListener(PythonJavaClass):
|
||||
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$NewIntentListener']
|
||||
__javacontext__ = 'app'
|
||||
|
||||
def __init__(self, callback, **kwargs):
|
||||
super(NewIntentListener, self).__init__(**kwargs)
|
||||
self.callback = callback
|
||||
|
||||
@java_method('(Landroid/content/Intent;)V')
|
||||
def onNewIntent(self, intent):
|
||||
self.callback(intent)
|
||||
|
||||
|
||||
class ActivityResultListener(PythonJavaClass):
|
||||
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$ActivityResultListener']
|
||||
__javacontext__ = 'app'
|
||||
|
||||
def __init__(self, callback):
|
||||
super(ActivityResultListener, self).__init__()
|
||||
self.callback = callback
|
||||
|
||||
@java_method('(IILandroid/content/Intent;)V')
|
||||
def onActivityResult(self, requestCode, resultCode, intent):
|
||||
self.callback(requestCode, resultCode, intent)
|
||||
|
||||
|
||||
def bind(**kwargs):
|
||||
for event, callback in kwargs.items():
|
||||
if event not in _callbacks:
|
||||
raise Exception('Unknown {!r} event'.format(event))
|
||||
elif event == 'on_new_intent':
|
||||
listener = NewIntentListener(callback)
|
||||
_activity.registerNewIntentListener(listener)
|
||||
_callbacks[event].append(listener)
|
||||
elif event == 'on_activity_result':
|
||||
listener = ActivityResultListener(callback)
|
||||
_activity.registerActivityResultListener(listener)
|
||||
_callbacks[event].append(listener)
|
||||
|
||||
|
||||
def unbind(**kwargs):
|
||||
for event, callback in kwargs.items():
|
||||
if event not in _callbacks:
|
||||
raise Exception('Unknown {!r} event'.format(event))
|
||||
else:
|
||||
for listener in _callbacks[event][:]:
|
||||
if listener.callback == callback:
|
||||
_callbacks[event].remove(listener)
|
||||
if event == 'on_new_intent':
|
||||
_activity.unregisterNewIntentListener(listener)
|
||||
elif event == 'on_activity_result':
|
||||
_activity.unregisterActivityResultListener(listener)
|
||||
|
||||
|
||||
print('YEAH! Done all the things!!! WOOOHOOO!!!!')
|
||||
@@ -0,0 +1,49 @@
|
||||
from os.path import join, dirname, abspath
|
||||
from pythonforandroid.toolchain import Bootstrap
|
||||
from setuptools import setup, find_packages
|
||||
from bootstrap import LBRYServiceBootstrap
|
||||
|
||||
|
||||
Bootstrap.bootstraps = {
|
||||
'lbry-service': LBRYServiceBootstrap()
|
||||
}
|
||||
|
||||
|
||||
setup(
|
||||
name='lbryservice',
|
||||
version='0.1',
|
||||
author="LBRY Inc.",
|
||||
author_email="hello@lbry.io",
|
||||
url="https://lbry.io",
|
||||
description="Android Service for LBRY Network.",
|
||||
license='MIT',
|
||||
python_requires='>=3.7',
|
||||
packages=find_packages(),
|
||||
package_data={'service': ['*.py']},
|
||||
options={
|
||||
'apk': {
|
||||
'dist_name': 'lbry-service',
|
||||
'bootstrap': 'lbry-service',
|
||||
'package': 'io.lbry.service',
|
||||
'permissions': ['INTERNET'],
|
||||
'requirements': ','.join([
|
||||
# needed by aiohttp
|
||||
'multidict', 'yarl', 'async_timeout', 'chardet',
|
||||
# minimum needed by torba:
|
||||
'aiohttp', 'coincurve', 'pbkdf2', 'cryptography', 'attrs',
|
||||
abspath(join(dirname(__file__), '..', '..', 'torba')),
|
||||
# minimum needed by lbrynet
|
||||
'aioupnp', 'appdirs', 'distro', 'base58', 'jsonrpc', 'protobuf',
|
||||
'msgpack', 'jsonschema', 'ecdsa', 'pyyaml', 'docopt',
|
||||
abspath(join(dirname(__file__), '..')),
|
||||
'genericndkbuild', 'pyjnius', 'sqlite3', 'python3'
|
||||
]),
|
||||
'android-api': '26',
|
||||
'ndk-api': '21',
|
||||
'ndk-version': 'r17c',
|
||||
'arch': 'armeabi-v7a',
|
||||
'sdk-dir': '/home/lex/projects/android',
|
||||
'ndk-dir': '/home/lex/projects/android/android-ndk-r17c/'
|
||||
}
|
||||
}
|
||||
)
|
||||
@@ -43,19 +43,13 @@
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "If provided only the balance for this account will be given",
|
||||
"description": "If provided only the balance for this account will be given. Otherwise default account.",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "address",
|
||||
"type": "str",
|
||||
"description": "If provided only the balance for this address will be given",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "include_unconfirmed",
|
||||
"type": "bool",
|
||||
"description": "Include unconfirmed",
|
||||
"name": "confirmations",
|
||||
"type": "int",
|
||||
"description": "Only include transactions with this many confirmed blocks.",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
@@ -83,18 +77,25 @@
|
||||
{
|
||||
"name": "account_decrypt",
|
||||
"description": "Decrypt an encrypted account, this will remove the wallet password",
|
||||
"arguments": [],
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id for the account to decrypt",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if wallet is decrypted, otherwise false"
|
||||
},
|
||||
{
|
||||
"name": "account_encrypt",
|
||||
"description": "Encrypt a wallet with a password, if the wallet is already encrypted this will update\nthe password",
|
||||
"description": "Encrypt an unencrypted account with a password",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "new_password",
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "password string to be used for encrypting wallet",
|
||||
"is_required": true
|
||||
"description": "id for the account to encrypt",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if wallet is decrypted, otherwise false"
|
||||
@@ -107,13 +108,13 @@
|
||||
"name": "to_account",
|
||||
"type": "str",
|
||||
"description": "send to this account",
|
||||
"is_required": true
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "from_account",
|
||||
"type": "str",
|
||||
"description": "spend from this account",
|
||||
"is_required": true
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "amount",
|
||||
@@ -155,13 +156,7 @@
|
||||
{
|
||||
"name": "confirmations",
|
||||
"type": "int",
|
||||
"description": "required confirmations (default: 6)",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "include_reserved",
|
||||
"type": "bool",
|
||||
"description": "include reserved UTXOs (default: false)",
|
||||
"description": "required confirmations (default: 0)",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
@@ -179,6 +174,19 @@
|
||||
],
|
||||
"returns": "(map) balance of account(s)"
|
||||
},
|
||||
{
|
||||
"name": "account_lock",
|
||||
"description": "Lock an unlocked account",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id for the account to lock",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if account is locked, otherwise false"
|
||||
},
|
||||
{
|
||||
"name": "account_max_address_gap",
|
||||
"description": "Finds ranges of consecutive addresses that are unused and returns the length\nof the longest such range: for change and receiving address chains. This is\nuseful to figure out ideal values to set for 'receiving_gap' and 'change_gap'\naccount settings.",
|
||||
@@ -205,6 +213,25 @@
|
||||
],
|
||||
"returns": "(map) details of removed account"
|
||||
},
|
||||
{
|
||||
"name": "account_send",
|
||||
"description": "Send the same number of credits to multiple addresses.",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "account to fund the transaction",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "broadcast",
|
||||
"type": "bool",
|
||||
"description": "actually broadcast the transaction, default: false.",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": ""
|
||||
},
|
||||
{
|
||||
"name": "account_set",
|
||||
"description": "Change various settings on an account.",
|
||||
@@ -259,10 +286,10 @@
|
||||
"description": "Unlock an encrypted account",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "password",
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "password for unlocking wallet",
|
||||
"is_required": true
|
||||
"description": "id for the account to unlock",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(bool) true if account is unlocked, otherwise false"
|
||||
@@ -295,23 +322,22 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to use",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "List of wallet addresses"
|
||||
},
|
||||
{
|
||||
"name": "address_public_key",
|
||||
"description": "Get public key from wallet address",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "address",
|
||||
"type": "str",
|
||||
"description": "address for which to get the public key",
|
||||
"is_required": true
|
||||
}
|
||||
],
|
||||
"returns": "(list) list of public keys associated with address.\n Could contain more than one public key if multisig."
|
||||
},
|
||||
{
|
||||
"name": "address_unused",
|
||||
"description": "Return an address containing no balance, will create\na new address if there is none.",
|
||||
@@ -535,7 +561,26 @@
|
||||
{
|
||||
"name": "channel_list",
|
||||
"description": "Get certificate claim infos for channels that can be published to",
|
||||
"arguments": [],
|
||||
"arguments": [
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id of the account to use",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet."
|
||||
},
|
||||
{
|
||||
@@ -553,6 +598,12 @@
|
||||
"type": "decimal",
|
||||
"description": "bid amount on the channel",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "id of the account to store channel",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }"
|
||||
@@ -584,6 +635,12 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to use",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "blocking",
|
||||
"type": "bool",
|
||||
"description": "wait until abandon is in mempool",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(dict) Dictionary containing result of the claim\n {\n success: (bool) True if txn is successful\n txid : (str) txid of resulting transaction\n }"
|
||||
@@ -641,6 +698,18 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to query",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) \"claim\", \"update\" , or \"support\"\n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the claim\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]"
|
||||
@@ -676,25 +745,6 @@
|
||||
],
|
||||
"returns": "(dict) Dictionary containing the transaction information\n {\n \"hex\": (str) raw transaction,\n \"inputs\": (list) inputs(dict) used for the transaction,\n \"outputs\": (list) outputs(dict) for the transaction,\n \"total_fee\": (int) fee in dewies,\n \"total_input\": (int) total of inputs in dewies,\n \"total_output\": (int) total of outputs in dewies(input - fees),\n \"txid\": (str) txid of the transaction,\n }"
|
||||
},
|
||||
{
|
||||
"name": "claim_renew",
|
||||
"description": "Renew claim(s) or support(s)",
|
||||
"arguments": [
|
||||
{
|
||||
"name": "outpoint",
|
||||
"type": "str",
|
||||
"description": "outpoint of the claim to renew",
|
||||
"is_required": true
|
||||
},
|
||||
{
|
||||
"name": "height",
|
||||
"type": "str",
|
||||
"description": "update claims expiring before or at this block height",
|
||||
"is_required": true
|
||||
}
|
||||
],
|
||||
"returns": "(dict) Dictionary where key is the the original claim's outpoint and\n value is the result of the renewal\n {\n outpoint:{\n\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n },\n }"
|
||||
},
|
||||
{
|
||||
"name": "claim_send_to_address",
|
||||
"description": "Send a name claim to an address",
|
||||
@@ -1190,6 +1240,18 @@
|
||||
"description": "claim id of the publisher channel, does not check for channel claim being in the wallet. This allows publishing to a channel where only the certificate private key is in the wallet.",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "channel_account_id",
|
||||
"type": "str",
|
||||
"description": "one or more account ids for accounts to look in for channel certificates, defaults to all accounts.",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "account_id",
|
||||
"type": "str",
|
||||
"description": "account to use for funding the transaction",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "claim_address",
|
||||
"type": "str",
|
||||
@@ -1363,7 +1425,7 @@
|
||||
"name": "status",
|
||||
"description": "Get daemon status",
|
||||
"arguments": [],
|
||||
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }"
|
||||
"returns": "(dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n },\n 'upnp': {\n 'aioupnp_version': (str),\n 'redirects': {\n <TCP | UDP>: (int) external_port,\n },\n 'gateway': (str) manufacturer and model,\n 'dht_redirect_set': (bool),\n 'peer_redirect_set': (bool),\n 'external_ip': (str) external ip address,\n }\n }"
|
||||
},
|
||||
{
|
||||
"name": "stop",
|
||||
@@ -1424,6 +1486,18 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to query",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) List of transactions\n\n {\n \"claim_info\": (list) claim info if in txn [{\n \"address\": (str) address of claim,\n \"balance_delta\": (float) bid amount,\n \"amount\": (float) claim amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"abandon_info\": (list) abandon info if in txn [{\n \"address\": (str) address of abandoned claim,\n \"balance_delta\": (float) returned amount,\n \"amount\": (float) claim amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"confirmations\": (int) number of confirmations for the txn,\n \"date\": (str) date and time of txn,\n \"fee\": (float) txn fee,\n \"support_info\": (list) support info if in txn [{\n \"address\": (str) address of support,\n \"balance_delta\": (float) support amount,\n \"amount\": (float) support amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"is_tip\": (bool),\n \"nout\": (int) nout\n }],\n \"timestamp\": (int) timestamp,\n \"txid\": (str) txn id,\n \"update_info\": (list) update info if in txn [{\n \"address\": (str) address of claim,\n \"balance_delta\": (float) credited/debited\n \"amount\": (float) absolute amount,\n \"claim_id\": (str) claim id,\n \"claim_name\": (str) claim name,\n \"nout\": (int) nout\n }],\n \"value\": (float) value of txn\n }"
|
||||
@@ -1450,6 +1524,18 @@
|
||||
"type": "str",
|
||||
"description": "id of the account to query",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page",
|
||||
"type": "int",
|
||||
"description": "page to return during paginating",
|
||||
"is_required": false
|
||||
},
|
||||
{
|
||||
"name": "page_size",
|
||||
"type": "int",
|
||||
"description": "number of items on page during pagination",
|
||||
"is_required": false
|
||||
}
|
||||
],
|
||||
"returns": "(list) List of unspent transaction outputs (UTXOs)\n [\n {\n \"address\": (str) the output address\n \"amount\": (float) unspent amount\n \"height\": (int) block height\n \"is_claim\": (bool) is the tx a claim\n \"is_coinbase\": (bool) is the tx a coinbase tx\n \"is_support\": (bool) is the tx a support\n \"is_update\": (bool) is the tx an update\n \"nout\": (int) nout of the output\n \"txid\": (str) txid of the output\n },\n ...\n ]"
|
||||
|
||||
@@ -17,9 +17,6 @@ dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
|
||||
use_auth_http: True
|
||||
use_https: True
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
|
||||
|
After Width: | Height: | Size: 7.4 KiB |
|
After Width: | Height: | Size: 19 KiB |
|
After Width: | Height: | Size: 1.2 KiB |
|
After Width: | Height: | Size: 2.6 KiB |
|
After Width: | Height: | Size: 6.1 KiB |
|
After Width: | Height: | Size: 97 KiB |
|
After Width: | Height: | Size: 1.1 KiB |
|
After Width: | Height: | Size: 361 KiB |
|
After Width: | Height: | Size: 5.3 KiB |
|
After Width: | Height: | Size: 15 KiB |
|
After Width: | Height: | Size: 31 KiB |
@@ -1,10 +1,4 @@
|
||||
import logging
|
||||
from lbrynet.custom_logger import install_logger
|
||||
|
||||
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.30.1rc2"
|
||||
__version__ = "0.31.0rc3"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
install_logger()
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -1 +0,0 @@
|
||||
from . import paths
|
||||
@@ -1,26 +0,0 @@
|
||||
# App root files dir
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29
|
||||
def android_files_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base internal storage path
|
||||
def android_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Base external (SD card, if present) storage path
|
||||
def android_external_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# Internal device storage (private app folder)
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_internal_storage_dir():
|
||||
return None
|
||||
|
||||
|
||||
# External (app folder on SD card, if present) storage
|
||||
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
|
||||
def android_app_external_storage_dir():
|
||||
return None
|
||||
@@ -1,166 +0,0 @@
|
||||
import binascii
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from lbrynet.p2p.BlobInfo import BlobInfo
|
||||
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
backend = default_backend()
|
||||
|
||||
|
||||
class CryptBlobInfo(BlobInfo):
|
||||
def __init__(self, blob_hash, blob_num, length, iv):
|
||||
super().__init__(blob_hash, blob_num, length)
|
||||
self.iv = iv
|
||||
|
||||
def get_dict(self):
|
||||
info = {
|
||||
"blob_num": self.blob_num,
|
||||
"length": self.length,
|
||||
"iv": self.iv.decode()
|
||||
}
|
||||
if self.blob_hash:
|
||||
info['blob_hash'] = self.blob_hash
|
||||
return info
|
||||
|
||||
|
||||
class StreamBlobDecryptor:
|
||||
def __init__(self, blob, key, iv, length):
|
||||
"""
|
||||
This class decrypts blob
|
||||
|
||||
blob - object which implements read() function.
|
||||
key = encryption_key
|
||||
iv = initialization vector
|
||||
blob_num = blob number (has no effect on encryption)
|
||||
length = length in bytes of blob
|
||||
"""
|
||||
self.blob = blob
|
||||
self.key = key
|
||||
self.iv = iv
|
||||
self.length = length
|
||||
self.buff = b''
|
||||
self.len_read = 0
|
||||
cipher = Cipher(AES(self.key), modes.CBC(self.iv), backend=backend)
|
||||
self.unpadder = PKCS7(AES.block_size).unpadder()
|
||||
self.cipher = cipher.decryptor()
|
||||
|
||||
def decrypt(self, write_func):
|
||||
"""
|
||||
Decrypt blob and write its content using write_func
|
||||
|
||||
write_func - function that takes decrypted string as
|
||||
argument and writes it somewhere
|
||||
|
||||
Returns:
|
||||
|
||||
deferred that returns after decrypting blob and writing content
|
||||
"""
|
||||
|
||||
def remove_padding(data):
|
||||
return self.unpadder.update(data) + self.unpadder.finalize()
|
||||
|
||||
def write_bytes():
|
||||
if self.len_read < self.length:
|
||||
num_bytes_to_decrypt = greatest_multiple(len(self.buff), (AES.block_size // 8))
|
||||
data_to_decrypt, self.buff = split(self.buff, num_bytes_to_decrypt)
|
||||
write_func(self.cipher.update(data_to_decrypt))
|
||||
|
||||
def finish_decrypt():
|
||||
bytes_left = len(self.buff) % (AES.block_size // 8)
|
||||
if bytes_left != 0:
|
||||
log.warning(self.buff[-1 * (AES.block_size // 8):].encode('hex'))
|
||||
raise Exception("blob %s has incorrect padding: %i bytes left" %
|
||||
(self.blob.blob_hash, bytes_left))
|
||||
data_to_decrypt, self.buff = self.buff, b''
|
||||
last_chunk = self.cipher.update(data_to_decrypt) + self.cipher.finalize()
|
||||
write_func(remove_padding(last_chunk))
|
||||
|
||||
|
||||
read_handle = self.blob.open_for_reading()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def decrypt_bytes():
|
||||
producer = FileBodyProducer(read_handle)
|
||||
buff = BytesIO()
|
||||
yield producer.startProducing(buff)
|
||||
self.buff = buff.getvalue()
|
||||
self.len_read += len(self.buff)
|
||||
write_bytes()
|
||||
finish_decrypt()
|
||||
|
||||
d = decrypt_bytes()
|
||||
return d
|
||||
|
||||
|
||||
class CryptStreamBlobMaker:
|
||||
def __init__(self, key, iv, blob_num, blob):
|
||||
"""
|
||||
This class encrypts data and writes it to a new blob
|
||||
|
||||
key = encryption_key
|
||||
iv = initialization vector
|
||||
blob_num = blob number (has no effect on encryption)
|
||||
blob = object which implements write(), close() function , close() function must
|
||||
be a deferred. (Will generally be of HashBlobCreator type)
|
||||
"""
|
||||
self.key = key
|
||||
self.iv = iv
|
||||
self.blob_num = blob_num
|
||||
self.blob = blob
|
||||
cipher = Cipher(AES(self.key), modes.CBC(self.iv), backend=backend)
|
||||
self.padder = PKCS7(AES.block_size).padder()
|
||||
self.cipher = cipher.encryptor()
|
||||
self.length = 0
|
||||
|
||||
def write(self, data):
|
||||
"""
|
||||
encrypt and write string data
|
||||
|
||||
Returns:
|
||||
tuple (done, num_bytes_to_write) where done is True if
|
||||
max bytes are written. num_bytes_to_write is the number
|
||||
of bytes that will be written from data in this call
|
||||
"""
|
||||
max_bytes_to_write = MAX_BLOB_SIZE - self.length - 1
|
||||
done = False
|
||||
if max_bytes_to_write <= len(data):
|
||||
num_bytes_to_write = max_bytes_to_write
|
||||
done = True
|
||||
else:
|
||||
num_bytes_to_write = len(data)
|
||||
data_to_write = data[:num_bytes_to_write]
|
||||
self.length += len(data_to_write)
|
||||
padded_data = self.padder.update(data_to_write)
|
||||
encrypted_data = self.cipher.update(padded_data)
|
||||
self.blob.write(encrypted_data)
|
||||
return done, num_bytes_to_write
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def close(self):
|
||||
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
|
||||
if self.length != 0:
|
||||
self.length += (AES.block_size // 8) - (self.length % (AES.block_size // 8))
|
||||
padded_data = self.padder.finalize()
|
||||
encrypted_data = self.cipher.update(padded_data) + self.cipher.finalize()
|
||||
self.blob.write(encrypted_data)
|
||||
|
||||
blob_hash = yield self.blob.close()
|
||||
log.debug("called the finished_callback from CryptStreamBlobMaker.close")
|
||||
blob = CryptBlobInfo(blob_hash, self.blob_num, self.length, binascii.hexlify(self.iv))
|
||||
defer.returnValue(blob)
|
||||
|
||||
|
||||
def greatest_multiple(a, b):
|
||||
"""return the largest value `c`, that is a multiple of `b` and is <= `a`"""
|
||||
return (a // b) * b
|
||||
|
||||
|
||||
def split(buff, cutoff):
|
||||
return buff[:cutoff], buff[cutoff:]
|
||||
@@ -1,149 +0,0 @@
|
||||
"""
|
||||
Utility for creating Crypt Streams, which are encrypted blobs and associated metadata.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from twisted.internet import defer
|
||||
from lbrynet.blob.CryptBlob import CryptStreamBlobMaker
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CryptStreamCreator:
|
||||
"""
|
||||
Create a new stream with blobs encrypted by a symmetric cipher.
|
||||
|
||||
Each blob is encrypted with the same key, but each blob has its
|
||||
own initialization vector which is associated with the blob when
|
||||
the blob is associated with the stream.
|
||||
"""
|
||||
|
||||
#implements(interfaces.IConsumer)
|
||||
|
||||
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
|
||||
"""@param blob_manager: Object that stores and provides access to blobs.
|
||||
@type blob_manager: BlobManager
|
||||
|
||||
@param name: the name of the stream, which will be presented to the user
|
||||
@type name: string
|
||||
|
||||
@param key: the raw AES key which will be used to encrypt the
|
||||
blobs. If None, a random key will be generated.
|
||||
@type key: string
|
||||
|
||||
@param iv_generator: a generator which yields initialization
|
||||
vectors for the blobs. Will be called once for each blob.
|
||||
@type iv_generator: a generator function which yields strings
|
||||
|
||||
@return: None
|
||||
"""
|
||||
self.blob_manager = blob_manager
|
||||
self.name = name
|
||||
self.key = key
|
||||
if iv_generator is None:
|
||||
self.iv_generator = self.random_iv_generator()
|
||||
else:
|
||||
self.iv_generator = iv_generator
|
||||
|
||||
self.stopped = True
|
||||
self.producer = None
|
||||
self.streaming = None
|
||||
self.blob_count = -1
|
||||
self.current_blob = None
|
||||
self.finished_deferreds = []
|
||||
|
||||
def registerProducer(self, producer, streaming):
|
||||
from twisted.internet import reactor
|
||||
|
||||
self.producer = producer
|
||||
self.streaming = streaming
|
||||
self.stopped = False
|
||||
if streaming is False:
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
def unregisterProducer(self):
|
||||
self.stopped = True
|
||||
self.producer = None
|
||||
|
||||
def _close_current_blob(self):
|
||||
# close the blob that was being written to
|
||||
# and save it to blob manager
|
||||
should_announce = self.blob_count == 0
|
||||
d = self.current_blob.close()
|
||||
d.addCallback(self._blob_finished)
|
||||
d.addCallback(lambda blob_info: self.blob_manager.creator_finished(blob_info,
|
||||
should_announce))
|
||||
self.finished_deferreds.append(d)
|
||||
self.current_blob = None
|
||||
|
||||
def stop(self):
|
||||
"""Stop creating the stream. Create the terminating zero-length blob."""
|
||||
log.debug("stop has been called for StreamCreator")
|
||||
self.stopped = True
|
||||
if self.current_blob is not None:
|
||||
self._close_current_blob()
|
||||
d = self._finalize()
|
||||
d.addCallback(lambda _: self._finished())
|
||||
return d
|
||||
|
||||
# TODO: move the stream creation process to its own thread and
|
||||
# remove the reactor from this process.
|
||||
def write(self, data):
|
||||
from twisted.internet import reactor
|
||||
self._write(data)
|
||||
if self.stopped is False and self.streaming is False:
|
||||
reactor.callLater(0, self.producer.resumeProducing)
|
||||
|
||||
@staticmethod
|
||||
def random_iv_generator():
|
||||
while 1:
|
||||
yield os.urandom(AES.block_size // 8)
|
||||
|
||||
def setup(self):
|
||||
"""Create the symmetric key if it wasn't provided"""
|
||||
|
||||
if self.key is None:
|
||||
self.key = os.urandom(AES.block_size // 8)
|
||||
|
||||
return defer.succeed(True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _finalize(self):
|
||||
"""
|
||||
Finalize a stream by adding an empty
|
||||
blob at the end, this is to indicate that
|
||||
the stream has ended. This empty blob is not
|
||||
saved to the blob manager
|
||||
"""
|
||||
|
||||
yield defer.DeferredList(self.finished_deferreds)
|
||||
self.blob_count += 1
|
||||
iv = next(self.iv_generator)
|
||||
final_blob = self._get_blob_maker(iv, self.blob_manager.get_blob_creator())
|
||||
stream_terminator = yield final_blob.close()
|
||||
terminator_info = yield self._blob_finished(stream_terminator)
|
||||
defer.returnValue(terminator_info)
|
||||
|
||||
def _write(self, data):
|
||||
while len(data) > 0:
|
||||
if self.current_blob is None:
|
||||
self.next_blob_creator = self.blob_manager.get_blob_creator()
|
||||
self.blob_count += 1
|
||||
iv = next(self.iv_generator)
|
||||
self.current_blob = self._get_blob_maker(iv, self.next_blob_creator)
|
||||
done, num_bytes_written = self.current_blob.write(data)
|
||||
data = data[num_bytes_written:]
|
||||
if done is True:
|
||||
self._close_current_blob()
|
||||
|
||||
def _get_blob_maker(self, iv, blob_creator):
|
||||
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
|
||||
|
||||
def _finished(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def _blob_finished(self, blob_info):
|
||||
raise NotImplementedError()
|
||||
@@ -1,131 +0,0 @@
|
||||
"""
|
||||
Utilities for turning plain files into LBRY Files.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
|
||||
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
|
||||
from lbrynet.blob.CryptStreamCreator import CryptStreamCreator
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileStreamCreator(CryptStreamCreator):
|
||||
"""
|
||||
A CryptStreamCreator which adds itself and its additional metadata to an EncryptedFileManager
|
||||
"""
|
||||
|
||||
def __init__(self, blob_manager, lbry_file_manager, stream_name=None,
|
||||
key=None, iv_generator=None):
|
||||
super().__init__(blob_manager, stream_name, key, iv_generator)
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.stream_hash = None
|
||||
self.blob_infos = []
|
||||
self.sd_info = None
|
||||
|
||||
def _blob_finished(self, blob_info):
|
||||
log.debug("length: %s", blob_info.length)
|
||||
self.blob_infos.append(blob_info.get_dict())
|
||||
return blob_info
|
||||
|
||||
def _finished(self):
|
||||
# calculate the stream hash
|
||||
self.stream_hash = get_stream_hash(
|
||||
hexlify(self.name.encode()).decode(), hexlify(self.key).decode(), hexlify(self.name.encode()).decode(),
|
||||
self.blob_infos
|
||||
)
|
||||
|
||||
# generate the sd info
|
||||
self.sd_info = format_sd_info(
|
||||
EncryptedFileStreamType, hexlify(self.name.encode()).decode(), hexlify(self.key).decode(),
|
||||
hexlify(self.name.encode()).decode(), self.stream_hash, self.blob_infos
|
||||
)
|
||||
|
||||
# sanity check
|
||||
validate_descriptor(self.sd_info)
|
||||
return defer.succeed(self.stream_hash)
|
||||
|
||||
|
||||
# TODO: this should be run its own thread. Encrypting a large file can
|
||||
# be very cpu intensive and there is no need to run that on the
|
||||
# main reactor thread. The FileSender mechanism that is used is
|
||||
# great when sending over the network, but this is all local so
|
||||
# we can simply read the file from the disk without needing to
|
||||
# involve reactor.
|
||||
@defer.inlineCallbacks
|
||||
def create_lbry_file(blob_manager, storage, payment_rate_manager, lbry_file_manager, file_name, file_handle,
|
||||
key=None, iv_generator=None):
|
||||
"""Turn a plain file into an LBRY File.
|
||||
|
||||
An LBRY File is a collection of encrypted blobs of data and the metadata that binds them
|
||||
together which, when decrypted and put back together according to the metadata, results
|
||||
in the original file.
|
||||
|
||||
The stream parameters that aren't specified are generated, the file is read and broken
|
||||
into chunks and encrypted, and then a stream descriptor file with the stream parameters
|
||||
and other metadata is written to disk.
|
||||
|
||||
@param session: An Session object.
|
||||
@type session: Session
|
||||
|
||||
@param lbry_file_manager: The EncryptedFileManager object this LBRY File will be added to.
|
||||
@type lbry_file_manager: EncryptedFileManager
|
||||
|
||||
@param file_name: The path to the plain file.
|
||||
@type file_name: string
|
||||
|
||||
@param file_handle: The file-like object to read
|
||||
@type file_handle: any file-like object which can be read by twisted.protocols.basic.FileSender
|
||||
|
||||
@param key: the raw AES key which will be used to encrypt the blobs. If None, a random key will
|
||||
be generated.
|
||||
@type key: string
|
||||
|
||||
@param iv_generator: a generator which yields initialization
|
||||
vectors for the blobs. Will be called once for each blob.
|
||||
@type iv_generator: a generator function which yields strings
|
||||
|
||||
@return: a Deferred which fires with the stream_hash of the LBRY File
|
||||
@rtype: Deferred which fires with hex-encoded string
|
||||
"""
|
||||
|
||||
base_file_name = os.path.basename(file_name)
|
||||
file_directory = os.path.dirname(file_handle.name)
|
||||
|
||||
lbry_file_creator = EncryptedFileStreamCreator(
|
||||
blob_manager, lbry_file_manager, base_file_name, key, iv_generator
|
||||
)
|
||||
|
||||
yield lbry_file_creator.setup()
|
||||
# TODO: Using FileSender isn't necessary, we can just read
|
||||
# straight from the disk. The stream creation process
|
||||
# should be in its own thread anyway so we don't need to
|
||||
# worry about interacting with the twisted reactor
|
||||
file_sender = FileSender()
|
||||
yield file_sender.beginFileTransfer(file_handle, lbry_file_creator)
|
||||
|
||||
log.debug("the file sender has triggered its deferred. stopping the stream writer")
|
||||
yield lbry_file_creator.stop()
|
||||
|
||||
log.debug("making the sd blob")
|
||||
sd_info = lbry_file_creator.sd_info
|
||||
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
|
||||
sd_hash = yield descriptor_writer.create_descriptor(sd_info)
|
||||
|
||||
log.debug("saving the stream")
|
||||
yield storage.store_stream(
|
||||
sd_info['stream_hash'], sd_hash, sd_info['stream_name'], sd_info['key'],
|
||||
sd_info['suggested_file_name'], sd_info['blobs']
|
||||
)
|
||||
log.debug("adding to the file manager")
|
||||
lbry_file = yield lbry_file_manager.add_published_file(
|
||||
sd_info['stream_hash'], sd_hash, hexlify(file_directory.encode()), payment_rate_manager,
|
||||
payment_rate_manager.min_blob_data_payment_rate
|
||||
)
|
||||
defer.returnValue(lbry_file)
|
||||
@@ -1,191 +0,0 @@
|
||||
"""
|
||||
Download LBRY Files from LBRYnet and save them to disk.
|
||||
"""
|
||||
import logging
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from lbrynet import conf
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
|
||||
from lbrynet.utils import short_hash
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def log_status(sd_hash, status):
|
||||
if status == ManagedEncryptedFileDownloader.STATUS_RUNNING:
|
||||
status_string = "running"
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_STOPPED:
|
||||
status_string = "stopped"
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
|
||||
status_string = "finished"
|
||||
else:
|
||||
status_string = "unknown"
|
||||
log.debug("stream %s is %s", short_hash(sd_hash), status_string)
|
||||
|
||||
|
||||
class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage, lbry_file_manager,
|
||||
payment_rate_manager, wallet, download_directory, file_name, stream_name, sd_hash, key,
|
||||
suggested_file_name, download_mirrors=None):
|
||||
super().__init__(
|
||||
stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name
|
||||
)
|
||||
self.sd_hash = sd_hash
|
||||
self.rowid = rowid
|
||||
self.suggested_file_name = unhexlify(suggested_file_name).decode()
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self._saving_status = False
|
||||
self.claim_id = None
|
||||
self.outpoint = None
|
||||
self.claim_name = None
|
||||
self.txid = None
|
||||
self.nout = None
|
||||
self.channel_claim_id = None
|
||||
self.channel_name = None
|
||||
self.metadata = None
|
||||
self.mirror = None
|
||||
if download_mirrors or conf.settings['download_mirrors']:
|
||||
self.mirror = HTTPBlobDownloader(
|
||||
self.blob_manager, servers=download_mirrors or conf.settings['download_mirrors']
|
||||
)
|
||||
|
||||
def set_claim_info(self, claim_info):
|
||||
self.claim_id = claim_info['claim_id']
|
||||
self.txid = claim_info['txid']
|
||||
self.nout = claim_info['nout']
|
||||
self.channel_claim_id = claim_info['channel_claim_id']
|
||||
self.outpoint = "%s:%i" % (self.txid, self.nout)
|
||||
self.claim_name = claim_info['name']
|
||||
self.channel_name = claim_info['channel_name']
|
||||
self.metadata = claim_info['value']['stream']['metadata']
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claim_info(self, include_supports=True):
|
||||
claim_info = yield self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
if claim_info:
|
||||
self.set_claim_info(claim_info)
|
||||
|
||||
defer.returnValue(claim_info)
|
||||
|
||||
@property
|
||||
def saving_status(self):
|
||||
return self._saving_status
|
||||
|
||||
def restore(self, status):
|
||||
if status == ManagedEncryptedFileDownloader.STATUS_RUNNING:
|
||||
# start returns self.finished_deferred
|
||||
# which fires when we've finished downloading the file
|
||||
# and we don't want to wait for the entire download
|
||||
self.start()
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_STOPPED:
|
||||
pass
|
||||
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
|
||||
self.completed = True
|
||||
else:
|
||||
raise Exception(f"Unknown status for stream {self.stream_hash}: {status}")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self, err=None, change_status=True):
|
||||
log.debug('Stopping download for stream %s', short_hash(self.stream_hash))
|
||||
if self.mirror:
|
||||
self.mirror.stop()
|
||||
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
yield EncryptedFileDownloader.stop(self, err=err)
|
||||
if change_status is True:
|
||||
status = yield self._save_status()
|
||||
defer.returnValue(status)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def status(self):
|
||||
blobs = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
|
||||
completed_blobs = yield self.blob_manager.completed_blobs(blob_hashes)
|
||||
num_blobs_completed = len(completed_blobs)
|
||||
num_blobs_known = len(blob_hashes)
|
||||
|
||||
if self.completed:
|
||||
status = "completed"
|
||||
elif self.stopped:
|
||||
status = "stopped"
|
||||
else:
|
||||
status = "running"
|
||||
defer.returnValue(EncryptedFileStatusReport(
|
||||
self.file_name, num_blobs_completed, num_blobs_known, status
|
||||
))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start(self):
|
||||
yield EncryptedFileSaver._start(self)
|
||||
status = yield self._save_status()
|
||||
log_status(self.sd_hash, status)
|
||||
if self.mirror:
|
||||
self.mirror.download_stream(self.stream_hash, self.sd_hash)
|
||||
defer.returnValue(status)
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
if self.completed is True:
|
||||
return "Download successful"
|
||||
else:
|
||||
return "Download stopped"
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _save_status(self):
|
||||
self._saving_status = True
|
||||
if self.completed is True:
|
||||
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
|
||||
elif self.stopped is True:
|
||||
status = ManagedEncryptedFileDownloader.STATUS_STOPPED
|
||||
else:
|
||||
status = ManagedEncryptedFileDownloader.STATUS_RUNNING
|
||||
status = yield self.lbry_file_manager.change_lbry_file_status(self, status)
|
||||
self._saving_status = False
|
||||
defer.returnValue(status)
|
||||
|
||||
def save_status(self):
|
||||
return self._save_status()
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
|
||||
class ManagedEncryptedFileDownloaderFactory:
|
||||
#implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, lbry_file_manager, blob_manager):
|
||||
self.lbry_file_manager = lbry_file_manager
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
def can_download(self, sd_validator):
|
||||
# TODO: add a sd_validator for non live streams, use it
|
||||
return True
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def make_downloader(self, metadata, data_rate, payment_rate_manager, download_directory, file_name=None,
|
||||
download_mirrors=None):
|
||||
stream_hash = yield save_sd_info(self.blob_manager,
|
||||
metadata.source_blob_hash,
|
||||
metadata.validator.raw_info)
|
||||
if file_name:
|
||||
file_name = hexlify(file_name.encode())
|
||||
hex_download_directory = hexlify(download_directory.encode())
|
||||
lbry_file = yield self.lbry_file_manager.add_downloaded_file(
|
||||
stream_hash, metadata.source_blob_hash, hex_download_directory, payment_rate_manager,
|
||||
data_rate, file_name=file_name, download_mirrors=download_mirrors
|
||||
)
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Save the file to disk"
|
||||
@@ -1,254 +0,0 @@
|
||||
"""
|
||||
Keep track of which LBRY Files are downloading and store their LBRY File specific metadata
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.reflector.reupload import reflect_file
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, get_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import AlreadyStoppedError
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CurrentlyStoppingError
|
||||
from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileManager:
|
||||
"""
|
||||
Keeps track of currently opened LBRY Files, their options, and
|
||||
their LBRY File specific metadata.
|
||||
"""
|
||||
# when reflecting files, reflect up to this many files at a time
|
||||
CONCURRENT_REFLECTS = 5
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, wallet, payment_rate_manager, storage, sd_identifier):
|
||||
self.auto_re_reflect = conf.settings['reflect_uploads'] and conf.settings['auto_re_reflect_interval'] > 0
|
||||
self.auto_re_reflect_interval = conf.settings['auto_re_reflect_interval']
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet = wallet
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.storage = storage
|
||||
# TODO: why is sd_identifier part of the file manager?
|
||||
self.sd_identifier = sd_identifier
|
||||
self.lbry_files = []
|
||||
self.lbry_file_reflector = task.LoopingCall(self.reflect_lbry_files)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
yield self._add_to_sd_identifier()
|
||||
yield self._start_lbry_files()
|
||||
log.info("Started file manager")
|
||||
|
||||
def get_lbry_file_status(self, lbry_file):
|
||||
return self.storage.get_lbry_file_status(lbry_file.rowid)
|
||||
|
||||
def set_lbry_file_data_payment_rate(self, lbry_file, new_rate):
|
||||
return self.storage(lbry_file.rowid, new_rate)
|
||||
|
||||
def change_lbry_file_status(self, lbry_file, status):
|
||||
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
|
||||
return self.storage.change_file_status(lbry_file.rowid, status)
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
|
||||
for lbry_file in self.lbry_files:
|
||||
ds.append(lbry_file.status())
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
|
||||
def filter_failures(status_reports):
|
||||
return [status_report for success, status_report in status_reports if success is True]
|
||||
|
||||
dl.addCallback(filter_failures)
|
||||
return dl
|
||||
|
||||
def _add_to_sd_identifier(self):
|
||||
downloader_factory = ManagedEncryptedFileDownloaderFactory(self, self.blob_manager)
|
||||
self.sd_identifier.add_stream_downloader_factory(
|
||||
EncryptedFileStreamType, downloader_factory)
|
||||
|
||||
def _get_lbry_file(self, rowid, stream_hash, payment_rate_manager, sd_hash, key,
|
||||
stream_name, file_name, download_directory, suggested_file_name, download_mirrors=None):
|
||||
return ManagedEncryptedFileDownloader(
|
||||
rowid,
|
||||
stream_hash,
|
||||
self.peer_finder,
|
||||
self.rate_limiter,
|
||||
self.blob_manager,
|
||||
self.storage,
|
||||
self,
|
||||
payment_rate_manager,
|
||||
self.wallet,
|
||||
download_directory,
|
||||
file_name,
|
||||
stream_name=stream_name,
|
||||
sd_hash=sd_hash,
|
||||
key=key,
|
||||
suggested_file_name=suggested_file_name,
|
||||
download_mirrors=download_mirrors
|
||||
)
|
||||
|
||||
def _start_lbry_file(self, file_info, payment_rate_manager, claim_info, download_mirrors=None):
|
||||
lbry_file = self._get_lbry_file(
|
||||
file_info['row_id'], file_info['stream_hash'], payment_rate_manager, file_info['sd_hash'],
|
||||
file_info['key'], file_info['stream_name'], file_info['file_name'], file_info['download_directory'],
|
||||
file_info['suggested_file_name'], download_mirrors
|
||||
)
|
||||
if claim_info:
|
||||
lbry_file.set_claim_info(claim_info)
|
||||
try:
|
||||
# restore will raise an Exception if status is unknown
|
||||
lbry_file.restore(file_info['status'])
|
||||
self.storage.content_claim_callbacks[lbry_file.stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
if len(self.lbry_files) % 500 == 0:
|
||||
log.info("Started %i files", len(self.lbry_files))
|
||||
except Exception:
|
||||
log.warning("Failed to start %i", file_info.get('rowid'))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start_lbry_files(self):
|
||||
files = yield self.storage.get_all_lbry_files()
|
||||
claim_infos = yield self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
prm = self.payment_rate_manager
|
||||
|
||||
log.info("Starting %i files", len(files))
|
||||
for file_info in files:
|
||||
claim_info = claim_infos.get(file_info['stream_hash'])
|
||||
self._start_lbry_file(file_info, prm, claim_info)
|
||||
|
||||
log.info("Started %i lbry files", len(self.lbry_files))
|
||||
if self.auto_re_reflect is True:
|
||||
safe_start_looping_call(self.lbry_file_reflector, self.auto_re_reflect_interval / 10)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop_lbry_file(self, lbry_file):
|
||||
def wait_for_finished(lbry_file, count=2):
|
||||
if count or lbry_file.saving_status is not False:
|
||||
return task.deferLater(reactor, 1, self._stop_lbry_file, lbry_file,
|
||||
count=count - 1)
|
||||
try:
|
||||
yield lbry_file.stop(change_status=False)
|
||||
self.lbry_files.remove(lbry_file)
|
||||
except CurrentlyStoppingError:
|
||||
yield wait_for_finished(lbry_file)
|
||||
except AlreadyStoppedError:
|
||||
pass
|
||||
finally:
|
||||
defer.returnValue(None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop_lbry_files(self):
|
||||
log.info("Stopping %i lbry files", len(self.lbry_files))
|
||||
yield defer.DeferredList([self._stop_lbry_file(lbry_file) for lbry_file in list(self.lbry_files)])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = stream_metadata['suggested_file_name']
|
||||
rowid = yield self.storage.save_published_file(
|
||||
stream_hash, file_name, download_directory, blob_data_rate, status
|
||||
)
|
||||
lbry_file = self._get_lbry_file(
|
||||
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
|
||||
stream_metadata['suggested_file_name'], download_mirrors=None
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info()
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
blob_data_rate=None, status=None, file_name=None, download_mirrors=None):
|
||||
status = status or ManagedEncryptedFileDownloader.STATUS_STOPPED
|
||||
payment_rate_manager = payment_rate_manager or self.payment_rate_manager
|
||||
blob_data_rate = blob_data_rate or payment_rate_manager.min_blob_data_payment_rate
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = file_name or stream_metadata['suggested_file_name']
|
||||
|
||||
# when we save the file we'll atomic touch the nearest file to the suggested file name
|
||||
# that doesn't yet exist in the download directory
|
||||
rowid = yield self.storage.save_downloaded_file(
|
||||
stream_hash, hexlify(os.path.basename(unhexlify(file_name))), download_directory, blob_data_rate
|
||||
)
|
||||
file_name = (yield self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
lbry_file = self._get_lbry_file(
|
||||
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
|
||||
stream_metadata['suggested_file_name'], download_mirrors
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info(include_supports=False)
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_lbry_file(self, lbry_file, delete_file=False):
|
||||
if lbry_file not in self.lbry_files:
|
||||
raise ValueError("Could not find that LBRY file")
|
||||
|
||||
def wait_for_finished(count=2):
|
||||
if count <= 0 or lbry_file.saving_status is False:
|
||||
return True
|
||||
else:
|
||||
return task.deferLater(reactor, 1, wait_for_finished, count=count - 1)
|
||||
|
||||
full_path = os.path.join(lbry_file.download_directory, lbry_file.file_name)
|
||||
|
||||
try:
|
||||
yield lbry_file.stop()
|
||||
except (AlreadyStoppedError, CurrentlyStoppingError):
|
||||
yield wait_for_finished()
|
||||
|
||||
self.lbry_files.remove(lbry_file)
|
||||
|
||||
if lbry_file.stream_hash in self.storage.content_claim_callbacks:
|
||||
del self.storage.content_claim_callbacks[lbry_file.stream_hash]
|
||||
|
||||
yield lbry_file.delete_data()
|
||||
yield self.storage.delete_stream(lbry_file.stream_hash)
|
||||
|
||||
if delete_file and os.path.isfile(full_path):
|
||||
os.remove(full_path)
|
||||
|
||||
defer.returnValue(True)
|
||||
|
||||
def toggle_lbry_file_running(self, lbry_file):
|
||||
"""Toggle whether a stream reader is currently running"""
|
||||
for l in self.lbry_files:
|
||||
if l == lbry_file:
|
||||
return l.toggle_running()
|
||||
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def reflect_lbry_files(self):
|
||||
sem = defer.DeferredSemaphore(self.CONCURRENT_REFLECTS)
|
||||
ds = []
|
||||
sd_hashes_to_reflect = yield self.storage.get_streams_to_re_reflect()
|
||||
for lbry_file in self.lbry_files:
|
||||
if lbry_file.sd_hash in sd_hashes_to_reflect:
|
||||
ds.append(sem.run(reflect_file, lbry_file))
|
||||
yield defer.DeferredList(ds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
safe_stop_looping_call(self.lbry_file_reflector)
|
||||
yield self._stop_lbry_files()
|
||||
log.info("Stopped encrypted file manager")
|
||||
defer.returnValue(True)
|
||||
@@ -1,6 +0,0 @@
|
||||
class EncryptedFileStatusReport:
|
||||
def __init__(self, name, num_completed, num_known, running_status):
|
||||
self.name = name
|
||||
self.num_completed = num_completed
|
||||
self.num_known = num_known
|
||||
self.running_status = running_status
|
||||
@@ -0,0 +1,6 @@
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
@@ -1,26 +1,27 @@
|
||||
import os
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
from twisted.internet import defer, threads
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
|
||||
import typing
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
|
||||
from lbrynet.cryptoutils import backend, get_lbry_hash_obj
|
||||
from lbrynet.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
|
||||
|
||||
from lbrynet.blob import MAX_BLOB_SIZE, blobhash_length
|
||||
from lbrynet.blob.blob_info import BlobInfo
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
from lbrynet.blob.reader import HashBlobReader
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
blobhash_length = get_lbry_hash_obj().digest_size * 2
|
||||
|
||||
|
||||
def is_valid_hashcharacter(char):
|
||||
def is_valid_hashcharacter(char: str) -> bool:
|
||||
return char in "0123456789abcdef"
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash):
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@@ -31,6 +32,16 @@ def is_valid_blobhash(blobhash):
|
||||
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
|
||||
|
||||
|
||||
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
padder = PKCS7(AES.block_size).padder()
|
||||
encryptor = cipher.encryptor()
|
||||
encrypted = encryptor.update(padder.update(unencrypted) + padder.finalize()) + encryptor.finalize()
|
||||
digest = get_lbry_hash_obj()
|
||||
digest.update(encrypted)
|
||||
return encrypted, digest.hexdigest()
|
||||
|
||||
|
||||
class BlobFile:
|
||||
"""
|
||||
A chunk of data available on the network which is specified by a hashsum
|
||||
@@ -40,188 +51,137 @@ class BlobFile:
|
||||
Also can be used for reading from blobs on the local filesystem
|
||||
"""
|
||||
|
||||
def __str__(self):
|
||||
return self.blob_hash[:16]
|
||||
|
||||
def __repr__(self):
|
||||
return '<{}({})>'.format(self.__class__.__name__, str(self))
|
||||
|
||||
def __init__(self, blob_dir, blob_hash, length=None):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, blob_hash: str,
|
||||
length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['BlobFile'], typing.Awaitable]] = None):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
self.loop = loop
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
self.writers = {} # {Peer: writer, finished_deferred}
|
||||
self._verified = False
|
||||
self.readers = 0
|
||||
self.blob_dir = blob_dir
|
||||
self.file_path = os.path.join(blob_dir, self.blob_hash)
|
||||
self.blob_write_lock = defer.DeferredLock()
|
||||
self.writers: typing.List[HashBlobWriter] = []
|
||||
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=loop)
|
||||
self.blob_write_lock = asyncio.Lock(loop=loop)
|
||||
if os.path.isfile(os.path.join(blob_dir, blob_hash)):
|
||||
length = length or int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
|
||||
self.length = length
|
||||
self.verified.set()
|
||||
self.finished_writing.set()
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
self.set_length(os.path.getsize(self.file_path))
|
||||
# This assumes that the hash of the blob has already been
|
||||
# checked as part of the blob creation process. It might
|
||||
# be worth having a function that checks the actual hash;
|
||||
# its probably too expensive to have that check be part of
|
||||
# this call.
|
||||
self._verified = True
|
||||
self.blob_completed_callback = blob_completed_callback
|
||||
|
||||
def open_for_writing(self, peer):
|
||||
def writer_finished(self, writer: HashBlobWriter):
|
||||
def callback(finished: asyncio.Future):
|
||||
try:
|
||||
error = finished.result()
|
||||
except Exception as err:
|
||||
error = err
|
||||
if writer in self.writers: # remove this download attempt
|
||||
self.writers.remove(writer)
|
||||
if not error: # the blob downloaded, cancel all the other download attempts and set the result
|
||||
while self.writers:
|
||||
other = self.writers.pop()
|
||||
other.finished.cancel()
|
||||
t = self.loop.create_task(self.save_verified_blob(writer))
|
||||
t.add_done_callback(lambda *_: self.finished_writing.set())
|
||||
return
|
||||
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
|
||||
log.error("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
|
||||
log.exception("something else")
|
||||
raise error
|
||||
return callback
|
||||
|
||||
async def save_verified_blob(self, writer):
|
||||
def _save_verified():
|
||||
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
|
||||
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
|
||||
if self.get_length() == len(writer.verified_bytes):
|
||||
with open(self.file_path, 'wb') as write_handle:
|
||||
write_handle.write(writer.verified_bytes)
|
||||
self.saved_verified_blob = True
|
||||
else:
|
||||
raise Exception("length mismatch")
|
||||
|
||||
if self.verified.is_set():
|
||||
return
|
||||
async with self.blob_write_lock:
|
||||
await self.loop.run_in_executor(None, _save_verified)
|
||||
if self.blob_completed_callback:
|
||||
await self.blob_completed_callback(self)
|
||||
self.verified.set()
|
||||
|
||||
def open_for_writing(self) -> HashBlobWriter:
|
||||
if os.path.exists(self.file_path):
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
self.writers.append(writer)
|
||||
fut.add_done_callback(self.writer_finished(writer))
|
||||
return writer
|
||||
|
||||
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
|
||||
"""
|
||||
open a blob file to be written by peer, supports concurrent
|
||||
writers, as long as they are from different peers.
|
||||
|
||||
returns tuple of (writer, finished_deferred)
|
||||
|
||||
writer - a file like object with a write() function, close() when finished
|
||||
finished_deferred - deferred that is fired when write is finished and returns
|
||||
a instance of itself as HashBlob
|
||||
Read and send the file to the writer and return the number of bytes sent
|
||||
"""
|
||||
if peer not in self.writers:
|
||||
log.debug("Opening %s to be written by %s", str(self), str(peer))
|
||||
finished_deferred = defer.Deferred()
|
||||
writer = HashBlobWriter(self.get_length, self.writer_finished)
|
||||
self.writers[peer] = (writer, finished_deferred)
|
||||
return writer, finished_deferred
|
||||
log.warning("Tried to download the same file twice simultaneously from the same peer")
|
||||
return None, None
|
||||
|
||||
def open_for_reading(self):
|
||||
"""
|
||||
open blob for reading
|
||||
with open(self.file_path, 'rb') as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle)
|
||||
|
||||
returns a file like object that can be read() from, and closed() when
|
||||
finished
|
||||
"""
|
||||
if self._verified is True:
|
||||
f = open(self.file_path, 'rb')
|
||||
reader = HashBlobReader(f, self.reader_finished)
|
||||
self.readers += 1
|
||||
return reader
|
||||
return None
|
||||
async def close(self):
|
||||
while self.writers:
|
||||
self.writers.pop().finished.cancel()
|
||||
|
||||
def delete(self):
|
||||
"""
|
||||
delete blob file from file system, prevent deletion
|
||||
if a blob is being read from or written to
|
||||
|
||||
returns a deferred that firesback when delete is completed
|
||||
"""
|
||||
if not self.writers and not self.readers:
|
||||
self._verified = False
|
||||
async def delete(self):
|
||||
await self.close()
|
||||
async with self.blob_write_lock:
|
||||
self.saved_verified_blob = False
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
def delete_from_file_system():
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
log.warning("An error occurred deleting %s: %s",
|
||||
str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
else:
|
||||
return defer.fail(Failure(
|
||||
ValueError("File is currently being read or written and cannot be deleted")))
|
||||
|
||||
@property
|
||||
def verified(self):
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
Protect verified from being modified by other classes.
|
||||
verified is True if a write to a blob has completed successfully,
|
||||
or a blob has been read to have the same length as specified
|
||||
in init
|
||||
Decrypt a BlobFile to plaintext bytes
|
||||
"""
|
||||
return self._verified
|
||||
|
||||
with open(self.file_path, "rb") as f:
|
||||
buff = f.read()
|
||||
if len(buff) != self.length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(buff) + decryptor.finalize()) + unpadder.finalize()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(cls, loop: asyncio.BaseEventLoop, blob_dir: str, key: bytes,
|
||||
iv: bytes, unencrypted: bytes, blob_num: int) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_dir, blob_hash, length)
|
||||
writer = blob.open_for_writing()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
|
||||
def set_length(self, length):
|
||||
if self.length is not None and length == self.length:
|
||||
return True
|
||||
return
|
||||
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
|
||||
self.length = length
|
||||
return True
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s",
|
||||
self.length, length)
|
||||
return False
|
||||
return
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
|
||||
|
||||
def get_length(self):
|
||||
return self.length
|
||||
|
||||
def get_is_verified(self):
|
||||
return self.verified
|
||||
|
||||
def is_downloading(self):
|
||||
if self.writers:
|
||||
return True
|
||||
return False
|
||||
|
||||
def reader_finished(self, reader):
|
||||
self.readers -= 1
|
||||
return defer.succeed(True)
|
||||
|
||||
def writer_finished(self, writer, err=None):
|
||||
def fire_finished_deferred():
|
||||
self._verified = True
|
||||
for p, (w, finished_deferred) in list(self.writers.items()):
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.callback(self)
|
||||
return True
|
||||
log.warning(
|
||||
"Somehow, the writer that was accepted as being valid was already removed: %s",
|
||||
writer)
|
||||
return False
|
||||
|
||||
def errback_finished_deferred(err):
|
||||
for p, (w, finished_deferred) in list(self.writers.items()):
|
||||
if w == writer:
|
||||
del self.writers[p]
|
||||
finished_deferred.errback(err)
|
||||
|
||||
def cancel_other_downloads():
|
||||
for p, (w, finished_deferred) in self.writers.items():
|
||||
w.close()
|
||||
|
||||
if err is None:
|
||||
if writer.len_so_far == self.length and writer.blob_hash == self.blob_hash:
|
||||
if self._verified is False:
|
||||
d = self.save_verified_blob(writer)
|
||||
d.addCallbacks(lambda _: fire_finished_deferred(), errback_finished_deferred)
|
||||
d.addCallback(lambda _: cancel_other_downloads())
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
fire_finished_deferred()
|
||||
else:
|
||||
if writer.len_so_far != self.length:
|
||||
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
|
||||
else:
|
||||
err_string = f"blob hash is {writer.blob_hash} vs expected {self.blob_hash}"
|
||||
errback_finished_deferred(Failure(InvalidDataError(err_string)))
|
||||
d = defer.succeed(None)
|
||||
else:
|
||||
errback_finished_deferred(err)
|
||||
d = defer.succeed(None)
|
||||
d.addBoth(lambda _: writer.close_handle())
|
||||
return d
|
||||
|
||||
def save_verified_blob(self, writer):
|
||||
# we cannot have multiple _save_verified_blob interrupting
|
||||
# each other, can happen since startProducing is a deferred
|
||||
return self.blob_write_lock.run(self._save_verified_blob, writer)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _save_verified_blob(self, writer):
|
||||
if self.saved_verified_blob is False:
|
||||
writer.write_handle.seek(0)
|
||||
out_path = os.path.join(self.blob_dir, self.blob_hash)
|
||||
producer = FileBodyProducer(writer.write_handle)
|
||||
yield producer.startProducing(open(out_path, 'wb'))
|
||||
self.saved_verified_blob = True
|
||||
defer.returnValue(True)
|
||||
else:
|
||||
raise DownloadCanceledError()
|
||||
return self.verified.is_set()
|
||||
@@ -0,0 +1,19 @@
|
||||
import typing
|
||||
|
||||
|
||||
class BlobInfo:
|
||||
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
self.iv = iv
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
'length': self.length,
|
||||
'blob_num': self.blob_num,
|
||||
'iv': self.iv,
|
||||
}
|
||||
if self.blob_hash: # non-terminator blobs have a blob hash
|
||||
d['blob_hash'] = self.blob_hash
|
||||
return d
|
||||
@@ -0,0 +1,89 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from sqlite3 import IntegrityError
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.stream.descriptor import StreamDescriptor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.protocol.data_store import DictDataStore
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobFileManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: SQLiteStorage,
|
||||
node_data_store: typing.Optional['DictDataStore'] = None):
|
||||
"""
|
||||
This class stores blobs on the hard disk
|
||||
|
||||
blob_dir - directory where blobs are stored
|
||||
storage - SQLiteStorage object
|
||||
"""
|
||||
self.loop = loop
|
||||
self.blob_dir = blob_dir
|
||||
self.storage = storage
|
||||
self._node_data_store = node_data_store
|
||||
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, BlobFile] = {}
|
||||
|
||||
async def setup(self) -> bool:
|
||||
raw_blob_hashes = await self.get_all_verified_blobs()
|
||||
self.completed_blob_hashes.update(raw_blob_hashes)
|
||||
return True
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = BlobFile(self.loop, self.blob_dir, blob_hash, length, self.blob_completed)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def get_stream_descriptor(self, sd_hash):
|
||||
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
|
||||
|
||||
async def blob_completed(self, blob: BlobFile):
|
||||
if blob.blob_hash is None:
|
||||
raise Exception("Blob hash is None")
|
||||
if not blob.length:
|
||||
raise Exception("Blob has a length of 0")
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
await self.storage.add_completed_blob(blob.blob_hash)
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = [self.get_blob(b) for b in blob_hashes]
|
||||
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
|
||||
|
||||
async def set_should_announce(self, blob_hash: str, should_announce: bool):
|
||||
now = self.loop.time()
|
||||
return await self.storage.set_should_announce(blob_hash, now, should_announce)
|
||||
|
||||
async def get_all_verified_blobs(self) -> typing.List[str]:
|
||||
blob_hashes = await self.storage.get_all_blob_hashes()
|
||||
return self.check_completed_blobs(blob_hashes)
|
||||
|
||||
async def delete_blobs(self, blob_hashes: typing.List[str]):
|
||||
bh_to_delete_from_db = []
|
||||
for blob_hash in blob_hashes:
|
||||
if not blob_hash:
|
||||
continue
|
||||
try:
|
||||
blob = self.get_blob(blob_hash)
|
||||
await blob.delete()
|
||||
bh_to_delete_from_db.append(blob_hash)
|
||||
except Exception as e:
|
||||
log.warning("Failed to delete blob file. Reason: %s", e)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
if blob_hash in self.blobs:
|
||||
del self.blobs[blob_hash]
|
||||
try:
|
||||
await self.storage.delete_blobs_from_db(bh_to_delete_from_db)
|
||||
except IntegrityError as err:
|
||||
if str(err) != "FOREIGN KEY constraint failed":
|
||||
raise err
|
||||
@@ -1,22 +0,0 @@
|
||||
import binascii
|
||||
from twisted.internet import defer
|
||||
from lbrynet.blob.CryptBlob import StreamBlobDecryptor
|
||||
|
||||
|
||||
class CryptBlobHandler:
|
||||
#implements(IBlobHandler)
|
||||
|
||||
def __init__(self, key, write_func):
|
||||
self.key = key
|
||||
self.write_func = write_func
|
||||
|
||||
######## IBlobHandler #########
|
||||
|
||||
def handle_blob(self, blob, blob_info):
|
||||
try:
|
||||
blob_decryptor = StreamBlobDecryptor(blob, self.key, binascii.unhexlify(blob_info.iv),
|
||||
blob_info.length)
|
||||
except ValueError as err:
|
||||
return defer.fail(err)
|
||||
d = blob_decryptor.decrypt(self.write_func)
|
||||
return d
|
||||
@@ -1,220 +0,0 @@
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from lbrynet.p2p.client.BlobRequester import BlobRequester
|
||||
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.p2p.client.DownloadManager import DownloadManager
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StartFailedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class AlreadyRunningError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class AlreadyStoppedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CurrentlyStoppingError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CurrentlyStartingError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class CryptStreamDownloader:
|
||||
|
||||
#implements(IStreamDownloader)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
|
||||
key, stream_name):
|
||||
"""Initialize a CryptStreamDownloader
|
||||
|
||||
@param peer_finder: An object which implements the IPeerFinder
|
||||
interface. Used to look up peers by a hashsum.
|
||||
|
||||
@param rate_limiter: An object which implements the IRateLimiter interface
|
||||
|
||||
@param blob_manager: A BlobManager object
|
||||
|
||||
@param payment_rate_manager: A NegotiatedPaymentRateManager object
|
||||
|
||||
@param wallet: An object which implements the IWallet interface
|
||||
|
||||
@return:
|
||||
|
||||
"""
|
||||
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
self.payment_rate_manager = payment_rate_manager
|
||||
self.wallet = wallet
|
||||
self.key = unhexlify(key)
|
||||
self.stream_name = unhexlify(stream_name).decode()
|
||||
self.completed = False
|
||||
self.stopped = True
|
||||
self.stopping = False
|
||||
self.starting = False
|
||||
self.download_manager = None
|
||||
self.finished_deferred = None
|
||||
self.points_paid = 0.0
|
||||
self.blob_requester = None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.stream_name)
|
||||
|
||||
def toggle_running(self):
|
||||
if self.stopped is True:
|
||||
return self.start()
|
||||
else:
|
||||
return self.stop()
|
||||
|
||||
def start(self):
|
||||
if self.starting is True:
|
||||
raise CurrentlyStartingError()
|
||||
if self.stopping is True:
|
||||
raise CurrentlyStoppingError()
|
||||
if self.stopped is False:
|
||||
raise AlreadyRunningError()
|
||||
assert self.download_manager is None
|
||||
self.starting = True
|
||||
self.completed = False
|
||||
self.finished_deferred = defer.Deferred()
|
||||
d = self._start()
|
||||
d.addCallback(lambda _: self.finished_deferred)
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self, err=None):
|
||||
if self.stopped is True:
|
||||
raise AlreadyStoppedError()
|
||||
if self.stopping is True:
|
||||
raise CurrentlyStoppingError()
|
||||
assert self.download_manager is not None
|
||||
self.stopping = True
|
||||
success = yield self.download_manager.stop_downloading()
|
||||
self.stopping = False
|
||||
if success is True:
|
||||
self.stopped = True
|
||||
self._remove_download_manager()
|
||||
yield self._fire_completed_deferred(err)
|
||||
|
||||
def _start_failed(self):
|
||||
|
||||
def set_stopped():
|
||||
self.stopped = True
|
||||
self.stopping = False
|
||||
self.starting = False
|
||||
|
||||
if self.download_manager is not None:
|
||||
d = self.download_manager.stop_downloading()
|
||||
d.addCallback(lambda _: self._remove_download_manager())
|
||||
else:
|
||||
d = defer.succeed(True)
|
||||
d.addCallback(lambda _: set_stopped())
|
||||
d.addCallback(lambda _: Failure(StartFailedError()))
|
||||
return d
|
||||
|
||||
def _start(self):
|
||||
|
||||
def check_start_succeeded(success):
|
||||
if success:
|
||||
self.starting = False
|
||||
self.stopped = False
|
||||
self.completed = False
|
||||
return True
|
||||
else:
|
||||
return self._start_failed()
|
||||
|
||||
self.download_manager = self._get_download_manager()
|
||||
d = self.download_manager.start_downloading()
|
||||
d.addCallbacks(check_start_succeeded)
|
||||
return d
|
||||
|
||||
def _get_download_manager(self):
|
||||
assert self.blob_requester is None
|
||||
download_manager = DownloadManager(self.blob_manager)
|
||||
# TODO: can we get rid of these circular references. I'm not
|
||||
# smart enough to handle thinking about the interactions
|
||||
# between them and have hope that there is a simpler way
|
||||
# to accomplish what we want
|
||||
download_manager.blob_info_finder = self._get_metadata_handler(download_manager)
|
||||
download_manager.progress_manager = self._get_progress_manager(download_manager)
|
||||
download_manager.blob_handler = self._get_blob_handler(download_manager)
|
||||
download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger()
|
||||
# blob_requester needs to be set before the connection manager is setup
|
||||
self.blob_requester = self._get_blob_requester(download_manager)
|
||||
download_manager.connection_manager = self._get_connection_manager(download_manager)
|
||||
return download_manager
|
||||
|
||||
def _remove_download_manager(self):
|
||||
self.download_manager.blob_info_finder = None
|
||||
self.download_manager.progress_manager = None
|
||||
self.download_manager.blob_handler = None
|
||||
self.download_manager.wallet_info_exchanger = None
|
||||
self.blob_requester = None
|
||||
self.download_manager.connection_manager = None
|
||||
self.download_manager = None
|
||||
|
||||
def _get_primary_request_creators(self, download_manager):
|
||||
return [self.blob_requester]
|
||||
|
||||
def _get_secondary_request_creators(self, download_manager):
|
||||
return [download_manager.wallet_info_exchanger]
|
||||
|
||||
def _get_metadata_handler(self, download_manager):
|
||||
pass
|
||||
|
||||
def _get_blob_requester(self, download_manager):
|
||||
return BlobRequester(self.blob_manager, self.peer_finder,
|
||||
self.payment_rate_manager, self.wallet,
|
||||
download_manager)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
def _get_write_func(self):
|
||||
pass
|
||||
|
||||
def _get_blob_handler(self, download_manager):
|
||||
return CryptBlobHandler(self.key, self._get_write_func())
|
||||
|
||||
def _get_connection_manager(self, download_manager):
|
||||
return ConnectionManager(self, self.rate_limiter,
|
||||
self._get_primary_request_creators(download_manager),
|
||||
self._get_secondary_request_creators(download_manager))
|
||||
|
||||
def _fire_completed_deferred(self, err=None):
|
||||
self.finished_deferred, d = None, self.finished_deferred
|
||||
if d is not None:
|
||||
if err is not None:
|
||||
d.errback(err)
|
||||
else:
|
||||
value = self._get_finished_deferred_callback_value()
|
||||
d.callback(value)
|
||||
else:
|
||||
log.debug("Not firing the completed deferred because d is None")
|
||||
|
||||
def _get_finished_deferred_callback_value(self):
|
||||
return None
|
||||
|
||||
def _finished_downloading(self, finished):
|
||||
if finished is True:
|
||||
self.completed = True
|
||||
return self.stop()
|
||||
|
||||
def insufficient_funds(self, err):
|
||||
return self.stop(err=err)
|
||||
@@ -1,198 +0,0 @@
|
||||
import os
|
||||
import logging
|
||||
import traceback
|
||||
from binascii import hexlify, unhexlify
|
||||
from twisted.internet import defer, threads
|
||||
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.Error import FileOpenError
|
||||
from lbrynet.blob.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
"""Classes which inherit from this class download LBRY files"""
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
storage, payment_rate_manager, wallet, key, stream_name, file_name):
|
||||
super().__init__(peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, key, stream_name)
|
||||
self.stream_hash = stream_hash
|
||||
self.storage = storage
|
||||
self.file_name = os.path.basename(unhexlify(file_name).decode())
|
||||
self._calculated_total_bytes = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_data(self):
|
||||
crypt_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
blob_hashes = [b.blob_hash for b in crypt_infos if b.blob_hash]
|
||||
sd_hash = yield self.storage.get_sd_blob_hash_for_stream(self.stream_hash)
|
||||
blob_hashes.append(sd_hash)
|
||||
yield self.blob_manager.delete_blobs(blob_hashes)
|
||||
|
||||
def stop(self, err=None):
|
||||
d = self._close_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager, download_manager)
|
||||
|
||||
def _start(self):
|
||||
d = self._setup_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader._start(self))
|
||||
return d
|
||||
|
||||
def _setup_output(self):
|
||||
pass
|
||||
|
||||
def _close_output(self):
|
||||
pass
|
||||
|
||||
def get_total_bytes(self):
|
||||
d = self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def calculate_size(blobs):
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
d.addCallback(calculate_size)
|
||||
return d
|
||||
|
||||
def get_total_bytes_cached(self):
|
||||
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
|
||||
if self.download_manager is None:
|
||||
return 0
|
||||
else:
|
||||
self._calculated_total_bytes = self.download_manager.calculate_total_bytes()
|
||||
return self._calculated_total_bytes
|
||||
|
||||
def get_bytes_left_to_output(self):
|
||||
if self.download_manager is not None:
|
||||
return self.download_manager.calculate_bytes_left_to_output()
|
||||
else:
|
||||
return 0
|
||||
|
||||
def get_bytes_left_to_download(self):
|
||||
if self.download_manager is not None:
|
||||
return self.download_manager.calculate_bytes_left_to_download()
|
||||
else:
|
||||
return 0
|
||||
|
||||
def _get_metadata_handler(self, download_manager):
|
||||
return EncryptedFileMetadataHandler(self.stream_hash,
|
||||
self.storage, download_manager)
|
||||
|
||||
|
||||
class EncryptedFileDownloaderFactory:
|
||||
#implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet):
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
self.storage = storage
|
||||
self.wallet = wallet
|
||||
|
||||
def can_download(self, sd_validator):
|
||||
return True
|
||||
|
||||
def make_downloader(self, metadata, options, payment_rate_manager, **kwargs):
|
||||
assert len(options) == 1
|
||||
data_rate = options[0]
|
||||
payment_rate_manager.min_blob_data_payment_rate = data_rate
|
||||
|
||||
def save_source_if_blob(stream_hash):
|
||||
return defer.succeed(metadata.source_blob_hash)
|
||||
|
||||
def create_downloader(stream_hash):
|
||||
downloader = self._make_downloader(stream_hash, payment_rate_manager,
|
||||
metadata.validator.raw_info)
|
||||
return defer.succeed(downloader)
|
||||
|
||||
d = save_sd_info(self.blob_manager, metadata.source_blob_hash, metadata.validator.raw_info)
|
||||
d.addCallback(save_source_if_blob)
|
||||
d.addCallback(create_downloader)
|
||||
return d
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
pass
|
||||
|
||||
|
||||
class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name):
|
||||
super().__init__(stream_hash, peer_finder, rate_limiter,
|
||||
blob_manager, storage, payment_rate_manager,
|
||||
wallet, key, stream_name, file_name)
|
||||
self.download_directory = unhexlify(download_directory).decode()
|
||||
self.file_written_to = os.path.join(self.download_directory, unhexlify(file_name).decode())
|
||||
self.file_handle = None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.file_written_to)
|
||||
|
||||
def stop(self, err=None):
|
||||
d = EncryptedFileDownloader.stop(self, err=err)
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager,
|
||||
download_manager)
|
||||
|
||||
def _setup_output(self):
|
||||
def open_file():
|
||||
if self.file_handle is None:
|
||||
file_written_to = os.path.join(self.download_directory, self.file_name)
|
||||
try:
|
||||
self.file_handle = open(file_written_to, 'wb')
|
||||
self.file_written_to = file_written_to
|
||||
except IOError:
|
||||
log.error(traceback.format_exc())
|
||||
raise FileOpenError(
|
||||
"Failed to open %s. Make sure you have permission to save files to that"
|
||||
" location." % file_written_to
|
||||
)
|
||||
return threads.deferToThread(open_file)
|
||||
|
||||
def _close_output(self):
|
||||
self.file_handle, file_handle = None, self.file_handle
|
||||
|
||||
def close_file():
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
return threads.deferToThread(close_file)
|
||||
|
||||
def _get_write_func(self):
|
||||
def write_func(data):
|
||||
if self.stopped is False and self.file_handle is not None:
|
||||
self.file_handle.write(data)
|
||||
return write_func
|
||||
|
||||
|
||||
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
|
||||
super().__init__(peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
self.download_directory = hexlify(download_directory.encode())
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
stream_name = stream_info.raw_info['stream_name']
|
||||
key = stream_info.raw_info['key']
|
||||
suggested_file_name = stream_info.raw_info['suggested_file_name']
|
||||
return EncryptedFileSaver(
|
||||
stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage, payment_rate_manager,
|
||||
self.wallet, self.download_directory, key=key, stream_name=stream_name, file_name=suggested_file_name
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_description():
|
||||
return "Save"
|
||||
@@ -1,40 +0,0 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EncryptedFileMetadataHandler:
|
||||
|
||||
def __init__(self, stream_hash, storage, download_manager):
|
||||
self.stream_hash = stream_hash
|
||||
self.storage = storage
|
||||
self.download_manager = download_manager
|
||||
self._final_blob_num = None
|
||||
|
||||
######### IMetadataHandler #########
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_initial_blobs(self):
|
||||
blob_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
formatted_infos = self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
defer.returnValue(formatted_infos)
|
||||
|
||||
def final_blob_num(self):
|
||||
return self._final_blob_num
|
||||
|
||||
######### internal calls #########
|
||||
|
||||
def _format_initial_blobs_for_download_manager(self, blob_infos):
|
||||
infos = []
|
||||
for i, crypt_blob in enumerate(blob_infos):
|
||||
if crypt_blob.blob_hash is not None and crypt_blob.length:
|
||||
infos.append(crypt_blob)
|
||||
else:
|
||||
if i != len(blob_infos) - 1:
|
||||
raise Exception("Invalid stream terminator: %i of %i" %
|
||||
(i, len(blob_infos) - 1))
|
||||
log.debug("Setting _final_blob_num to %s", str(crypt_blob.blob_num - 1))
|
||||
self._final_blob_num = crypt_blob.blob_num - 1
|
||||
return infos
|
||||
@@ -1,47 +0,0 @@
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamDescriptorValidator
|
||||
from lbrynet.p2p.DownloadOption import DownloadOption, DownloadOptionChoice
|
||||
|
||||
|
||||
def add_lbry_file_to_sd_identifier(sd_identifier):
|
||||
sd_identifier.add_stream_type(EncryptedFileStreamType, EncryptedFileStreamDescriptorValidator,
|
||||
EncryptedFileOptions())
|
||||
|
||||
|
||||
class EncryptedFileOptions:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def get_downloader_options(self, sd_validator, payment_rate_manager):
|
||||
prm = payment_rate_manager
|
||||
|
||||
def get_default_data_rate_description():
|
||||
if prm.base.min_blob_data_payment_rate is None:
|
||||
return "Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)
|
||||
else:
|
||||
return "%f LBC/MB" % prm.base.min_blob_data_payment_rate
|
||||
|
||||
rate_choices = []
|
||||
rate_choices.append(DownloadOptionChoice(
|
||||
prm.base.min_blob_data_payment_rate,
|
||||
"No change - %s" % get_default_data_rate_description(),
|
||||
"No change - %s" % get_default_data_rate_description()))
|
||||
if prm.base.min_blob_data_payment_rate is not None:
|
||||
rate_choices.append(DownloadOptionChoice(
|
||||
None,
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate),
|
||||
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)))
|
||||
rate_choices.append(DownloadOptionChoice(float,
|
||||
"Enter rate in LBC/MB",
|
||||
"Enter rate in LBC/MB"))
|
||||
|
||||
options = [
|
||||
DownloadOption(
|
||||
rate_choices,
|
||||
"Rate which will be paid for data",
|
||||
"data payment rate",
|
||||
prm.base.min_blob_data_payment_rate,
|
||||
get_default_data_rate_description()
|
||||
),
|
||||
]
|
||||
return options
|
||||
@@ -1,51 +0,0 @@
|
||||
import os
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobFileCreator:
|
||||
"""
|
||||
This class is used to create blobs on the local filesystem
|
||||
when we do not know the blob hash beforehand (i.e, when creating
|
||||
a new stream)
|
||||
"""
|
||||
def __init__(self, blob_dir):
|
||||
self.blob_dir = blob_dir
|
||||
self.buffer = BytesIO()
|
||||
self._is_open = True
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.blob_hash = None
|
||||
self.length = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def close(self):
|
||||
self.length = self.len_so_far
|
||||
self.blob_hash = self._hashsum.hexdigest()
|
||||
if self.blob_hash and self._is_open and self.length > 0:
|
||||
# do not save 0 length files (empty tail blob in streams)
|
||||
# or if its been closed already
|
||||
self.buffer.seek(0)
|
||||
out_path = os.path.join(self.blob_dir, self.blob_hash)
|
||||
producer = FileBodyProducer(self.buffer)
|
||||
yield producer.startProducing(open(out_path, 'wb'))
|
||||
self._is_open = False
|
||||
if self.length > 0:
|
||||
defer.returnValue(self.blob_hash)
|
||||
else:
|
||||
# 0 length files (empty tail blob in streams )
|
||||
# must return None as their blob_hash for
|
||||
# it to be saved properly by EncryptedFileMetadataManagers
|
||||
defer.returnValue(None)
|
||||
|
||||
def write(self, data):
|
||||
if not self._is_open:
|
||||
raise IOError
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
self.buffer.write(data)
|
||||
@@ -1,30 +0,0 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobReader:
|
||||
"""
|
||||
This is a file like reader class that supports
|
||||
read(size) and close()
|
||||
"""
|
||||
def __init__(self, read_handle, finished_cb):
|
||||
self.finished_cb = finished_cb
|
||||
self.finished_cb_d = None
|
||||
self.read_handle = read_handle
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
log.warning("Garbage collection was called, but reader for %s was not closed yet",
|
||||
self.read_handle.name)
|
||||
self.close()
|
||||
|
||||
def read(self, size=-1):
|
||||
return self.read_handle.read(size)
|
||||
|
||||
def close(self):
|
||||
# if we've already closed and called finished_cb, do nothing
|
||||
if self.finished_cb_d is not None:
|
||||
return
|
||||
self.read_handle.close()
|
||||
self.finished_cb_d = self.finished_cb(self)
|
||||
@@ -1,58 +1,71 @@
|
||||
import typing
|
||||
import logging
|
||||
import asyncio
|
||||
from io import BytesIO
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
|
||||
from lbrynet.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobWriter:
|
||||
def __init__(self, length_getter, finished_cb):
|
||||
self.write_handle = BytesIO()
|
||||
self.length_getter = length_getter
|
||||
self.finished_cb = finished_cb
|
||||
self.finished_cb_d = None
|
||||
def __init__(self, expected_blob_hash: str, get_length: typing.Callable[[], int],
|
||||
finished: asyncio.Future):
|
||||
self.expected_blob_hash = expected_blob_hash
|
||||
self.get_length = get_length
|
||||
self.buffer = BytesIO()
|
||||
self.finished = finished
|
||||
self.finished.add_done_callback(lambda *_: self.close_handle())
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
self.verified_bytes = b''
|
||||
|
||||
def __del__(self):
|
||||
if self.finished_cb_d is None:
|
||||
if self.buffer is not None:
|
||||
log.warning("Garbage collection was called, but writer was not closed yet")
|
||||
self.close()
|
||||
self.close_handle()
|
||||
|
||||
@property
|
||||
def blob_hash(self):
|
||||
def calculate_blob_hash(self) -> str:
|
||||
return self._hashsum.hexdigest()
|
||||
|
||||
def write(self, data):
|
||||
if self.write_handle is None:
|
||||
def closed(self):
|
||||
return self.buffer is None or self.buffer.closed
|
||||
|
||||
def write(self, data: bytes):
|
||||
expected_length = self.get_length()
|
||||
if not expected_length:
|
||||
raise IOError("unknown blob length")
|
||||
if self.buffer is None:
|
||||
log.warning("writer has already been closed")
|
||||
if not self.finished.done():
|
||||
self.finished.cancel()
|
||||
return
|
||||
raise IOError('I/O operation on closed file')
|
||||
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > self.length_getter():
|
||||
self.finished_cb_d = self.finished_cb(
|
||||
self,
|
||||
Failure(InvalidDataError("Length so far is greater than the expected length."
|
||||
" %s to %s" % (self.len_so_far,
|
||||
self.length_getter()))))
|
||||
else:
|
||||
self.write_handle.write(data)
|
||||
if self.len_so_far == self.length_getter():
|
||||
self.finished_cb_d = self.finished_cb(self)
|
||||
if self.len_so_far > expected_length:
|
||||
self.close_handle()
|
||||
self.finished.set_result(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
return
|
||||
self.buffer.write(data)
|
||||
if self.len_so_far == expected_length:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.close_handle()
|
||||
self.finished.set_result(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
return
|
||||
self.buffer.seek(0)
|
||||
self.verified_bytes = self.buffer.read()
|
||||
self.close_handle()
|
||||
if self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(None)
|
||||
|
||||
def close_handle(self):
|
||||
if self.write_handle is not None:
|
||||
self.write_handle.close()
|
||||
self.write_handle = None
|
||||
|
||||
def close(self, reason=None):
|
||||
# if we've already called finished_cb because we either finished writing
|
||||
# or closed already, do nothing
|
||||
if self.finished_cb_d is not None:
|
||||
return
|
||||
if reason is None:
|
||||
reason = Failure(DownloadCanceledError())
|
||||
self.finished_cb_d = self.finished_cb(self, reason)
|
||||
if self.buffer is not None:
|
||||
self.buffer.close()
|
||||
self.buffer = None
|
||||
@@ -0,0 +1,168 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.writer import HashBlobWriter
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10):
|
||||
self.loop = loop
|
||||
self.peer_port: typing.Optional[int] = None
|
||||
self.peer_address: typing.Optional[str] = None
|
||||
self.peer_timeout = peer_timeout
|
||||
self.transport: asyncio.Transport = None
|
||||
|
||||
self.writer: 'HashBlobWriter' = None
|
||||
self.blob: 'BlobFile' = None
|
||||
self.download_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: asyncio.Future = None
|
||||
self._request_lock = asyncio.Lock(loop=self.loop)
|
||||
|
||||
def handle_data_received(self, data: bytes):
|
||||
if self.transport.is_closing():
|
||||
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()):
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
|
||||
response = BlobResponse.deserialize(data)
|
||||
|
||||
if response.responses and self.blob:
|
||||
blob_response = response.get_blob_response()
|
||||
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
|
||||
self.blob.set_length(blob_response.length)
|
||||
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
|
||||
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
|
||||
return
|
||||
if response.responses:
|
||||
self._response_fut.set_result(response)
|
||||
if response.blob_data and self.writer and not self.writer.closed():
|
||||
self._blob_bytes_received += len(response.blob_data)
|
||||
try:
|
||||
self.writer.write(response.blob_data)
|
||||
except IOError as err:
|
||||
log.error("error downloading blob: %s", err)
|
||||
|
||||
def data_received(self, data):
|
||||
try:
|
||||
return self.handle_data_received(data)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
|
||||
async def _download_blob(self) -> typing.Tuple[bool, bool]:
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
try:
|
||||
self.transport.write(request.serialize())
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("blob not in availability response")
|
||||
return False, True
|
||||
elif availability_response.available_blobs and \
|
||||
availability_response.available_blobs != [self.blob.blob_hash]:
|
||||
log.warning("blob availability response doesn't match our request")
|
||||
return False, False
|
||||
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
|
||||
log.warning("data rate rejected")
|
||||
return False, False
|
||||
if not blob_response or blob_response.error:
|
||||
log.warning("blob cant be downloaded from this peer")
|
||||
return False, True
|
||||
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
|
||||
log.warning("incoming blob hash mismatch")
|
||||
return False, False
|
||||
if self.blob.length is not None and self.blob.length != blob_response.length:
|
||||
log.warning("incoming blob unexpected length")
|
||||
return False, False
|
||||
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
|
||||
f" timeout in {self.peer_timeout}"
|
||||
log.info(msg)
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
await self.blob.finished_writing.wait()
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}"
|
||||
log.info(msg)
|
||||
return True, True
|
||||
except asyncio.CancelledError:
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
return False, False
|
||||
finally:
|
||||
await self.close()
|
||||
|
||||
async def close(self):
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
if self.blob:
|
||||
await self.blob.close()
|
||||
self.download_running.clear()
|
||||
self._response_fut = None
|
||||
self.writer = None
|
||||
self.blob = None
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
self.transport = None
|
||||
|
||||
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[bool, bool]:
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
async with self._request_lock:
|
||||
try:
|
||||
if self.download_running.is_set():
|
||||
log.info("wait for download already running")
|
||||
await self.download_running.wait()
|
||||
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
|
||||
self.download_running.set()
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError:
|
||||
log.error("race happened")
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
return False, True
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, False
|
||||
except asyncio.CancelledError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return False, True
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
self.transport = transport
|
||||
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
|
||||
# log.info("connection made to %s: %s", self.peer_address, transport)
|
||||
|
||||
def connection_lost(self, reason):
|
||||
# log.info("connection lost to %s (reason: %s)", self.peer_address, reason)
|
||||
self.transport = None
|
||||
self.loop.create_task(self.close())
|
||||
|
||||
|
||||
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol: 'BlobExchangeClientProtocol',
|
||||
address: str, tcp_port: int, peer_connect_timeout: float) -> typing.Tuple[bool, bool]:
|
||||
"""
|
||||
Returns [<downloaded blob>, <keep connection>]
|
||||
"""
|
||||
if blob.get_is_verified():
|
||||
return False, True
|
||||
if blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return False, True
|
||||
try:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
return await protocol.download_blob(blob)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return False, False
|
||||
@@ -0,0 +1,132 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet import conf
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def drain_into(a: list, b: list):
|
||||
while a:
|
||||
b.append(a.pop())
|
||||
|
||||
|
||||
class BlobDownloader: # TODO: refactor to be the base class used by StreamDownloader
|
||||
"""A single blob downloader"""
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', config: conf.Config):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.new_peer_event = asyncio.Event(loop=self.loop)
|
||||
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
|
||||
self.running_download_requests: typing.List[asyncio.Task] = []
|
||||
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.blob: 'BlobFile' = None
|
||||
self.blob_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.blob_download_timeout = config.get('blob_download_timeout')
|
||||
self.peer_connect_timeout = config.get('peer_connect_timeout')
|
||||
self.max_connections = config.get('max_connections_per_stream')
|
||||
|
||||
async def _request_blob(self, peer: 'KademliaPeer'):
|
||||
if self.blob.get_is_verified():
|
||||
log.info("already verified")
|
||||
return
|
||||
if peer not in self.active_connections:
|
||||
log.warning("not active, adding: %s", str(peer))
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
protocol = self.active_connections[peer]
|
||||
success, keep_connection = await request_blob(self.loop, self.blob, protocol, peer.address, peer.tcp_port,
|
||||
self.peer_connect_timeout)
|
||||
await protocol.close()
|
||||
if not keep_connection:
|
||||
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
if peer in self.active_connections:
|
||||
async with self.lock:
|
||||
del self.active_connections[peer]
|
||||
return
|
||||
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
|
||||
def _update_requests(self):
|
||||
self.new_peer_event.clear()
|
||||
if self.blob.blob_hash not in self.requested_from:
|
||||
self.requested_from[self.blob.blob_hash] = {}
|
||||
to_add = []
|
||||
for peer in self.active_connections.keys():
|
||||
if peer not in self.requested_from[self.blob.blob_hash] and peer not in to_add:
|
||||
to_add.append(peer)
|
||||
if to_add or self.running_download_requests:
|
||||
log.info("adding download probes for %i peers to %i already active",
|
||||
min(len(to_add), 8 - len(self.running_download_requests)),
|
||||
len(self.running_download_requests))
|
||||
else:
|
||||
log.info("downloader idle...")
|
||||
for peer in to_add:
|
||||
if len(self.running_download_requests) >= 8:
|
||||
break
|
||||
task = self.loop.create_task(self._request_blob(peer))
|
||||
self.requested_from[self.blob.blob_hash][peer] = task
|
||||
self.running_download_requests.append(task)
|
||||
|
||||
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if peer not in self.active_connections:
|
||||
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
|
||||
added += 1
|
||||
if added:
|
||||
if not self.new_peer_event.is_set():
|
||||
log.info("added %i new peers", len(peers))
|
||||
self.new_peer_event.set()
|
||||
|
||||
async def _accumulate_connections(self, node: 'Node'):
|
||||
try:
|
||||
async with node.stream_peer_search_junction(self.blob_queue) as search_junction:
|
||||
async for peers in search_junction:
|
||||
if not isinstance(peers, list): # TODO: what's up with this?
|
||||
log.error("not a list: %s %s", peers, str(type(peers)))
|
||||
else:
|
||||
self._add_peer_protocols(peers)
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def get_blob(self, blob_hash: str, node: 'Node') -> 'BlobFile':
|
||||
self.blob = self.blob_manager.get_blob(blob_hash)
|
||||
if self.blob.get_is_verified():
|
||||
return self.blob
|
||||
accumulator = self.loop.create_task(self._accumulate_connections(node))
|
||||
self.blob_queue.put_nowait(blob_hash)
|
||||
try:
|
||||
while not self.blob.get_is_verified():
|
||||
if len(self.running_download_requests) < self.max_connections:
|
||||
self._update_requests()
|
||||
|
||||
# drain the tasks into a temporary list
|
||||
download_tasks = []
|
||||
drain_into(self.running_download_requests, download_tasks)
|
||||
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
|
||||
|
||||
# wait for a new peer to be added or for a download attempt to finish
|
||||
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
|
||||
loop=self.loop)
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
if self.blob.get_is_verified():
|
||||
if got_new_peer and not got_new_peer.done():
|
||||
got_new_peer.cancel()
|
||||
drain_tasks(download_tasks)
|
||||
return self.blob
|
||||
except asyncio.CancelledError:
|
||||
drain_tasks(self.running_download_requests)
|
||||
raise
|
||||
finally:
|
||||
if accumulator and not accumulator.done():
|
||||
accumulator.cancel()
|
||||
@@ -0,0 +1,289 @@
|
||||
import typing
|
||||
import json
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobMessage:
|
||||
key = ''
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class BlobPriceRequest(BlobMessage):
|
||||
key = 'blob_data_payment_rate'
|
||||
|
||||
def __init__(self, blob_data_payment_rate: float, **kwargs) -> None:
|
||||
self.blob_data_payment_rate = blob_data_payment_rate
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.blob_data_payment_rate
|
||||
}
|
||||
|
||||
|
||||
class BlobPriceResponse(BlobMessage):
|
||||
key = 'blob_data_payment_rate'
|
||||
rate_accepted = 'RATE_ACCEPTED'
|
||||
rate_too_low = 'RATE_TOO_LOW'
|
||||
rate_unset = 'RATE_UNSET'
|
||||
|
||||
def __init__(self, blob_data_payment_rate: str, **kwargs) -> None:
|
||||
if blob_data_payment_rate not in (self.rate_accepted, self.rate_too_low, self.rate_unset):
|
||||
raise ValueError(blob_data_payment_rate)
|
||||
self.blob_data_payment_rate = blob_data_payment_rate
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.blob_data_payment_rate
|
||||
}
|
||||
|
||||
|
||||
class BlobAvailabilityRequest(BlobMessage):
|
||||
key = 'requested_blobs'
|
||||
|
||||
def __init__(self, requested_blobs: typing.List[str], lbrycrd_address: typing.Optional[bool] = True,
|
||||
**kwargs) -> None:
|
||||
assert len(requested_blobs)
|
||||
self.requested_blobs = requested_blobs
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.requested_blobs,
|
||||
'lbrycrd_address': self.lbrycrd_address
|
||||
}
|
||||
|
||||
|
||||
class BlobAvailabilityResponse(BlobMessage):
|
||||
key = 'available_blobs'
|
||||
|
||||
def __init__(self, available_blobs: typing.List[str], lbrycrd_address: typing.Optional[str] = True,
|
||||
**kwargs) -> None:
|
||||
self.available_blobs = available_blobs
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
self.key: self.available_blobs
|
||||
}
|
||||
if self.lbrycrd_address:
|
||||
d['lbrycrd_address'] = self.lbrycrd_address
|
||||
return d
|
||||
|
||||
|
||||
class BlobDownloadRequest(BlobMessage):
|
||||
key = 'requested_blob'
|
||||
|
||||
def __init__(self, requested_blob: str, **kwargs) -> None:
|
||||
self.requested_blob = requested_blob
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.requested_blob
|
||||
}
|
||||
|
||||
|
||||
class BlobDownloadResponse(BlobMessage):
|
||||
key = 'incoming_blob'
|
||||
|
||||
def __init__(self, **response: typing.Dict) -> None:
|
||||
incoming_blob = response[self.key]
|
||||
self.error = None
|
||||
self.incoming_blob = None
|
||||
if 'error' in incoming_blob:
|
||||
self.error = incoming_blob['error']
|
||||
else:
|
||||
self.incoming_blob = {'blob_hash': incoming_blob['blob_hash'], 'length': incoming_blob['length']}
|
||||
self.length = None if not self.incoming_blob else self.incoming_blob['length']
|
||||
self.blob_hash = None if not self.incoming_blob else self.incoming_blob['blob_hash']
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key if not self.error else 'error': self.incoming_blob or self.error,
|
||||
}
|
||||
|
||||
|
||||
class BlobPaymentAddressRequest(BlobMessage):
|
||||
key = 'lbrycrd_address'
|
||||
|
||||
def __init__(self, lbrycrd_address: str, **kwargs) -> None:
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.lbrycrd_address
|
||||
}
|
||||
|
||||
|
||||
class BlobPaymentAddressResponse(BlobPaymentAddressRequest):
|
||||
pass
|
||||
|
||||
|
||||
class BlobErrorResponse(BlobMessage):
|
||||
key = 'error'
|
||||
|
||||
def __init__(self, error: str, **kwargs) -> None:
|
||||
self.error = error
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.error
|
||||
}
|
||||
|
||||
|
||||
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
|
||||
BlobPaymentAddressRequest]
|
||||
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
|
||||
BlobErrorResponse, BlobPaymentAddressResponse]
|
||||
|
||||
|
||||
def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[typing.Dict], bytes]:
|
||||
# scenarios:
|
||||
# <json>
|
||||
# <blob bytes>
|
||||
# <json><blob bytes>
|
||||
|
||||
extra_data = b''
|
||||
response = None
|
||||
curr_pos = 0
|
||||
while True:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren == -1:
|
||||
break
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
if not isinstance(response, dict):
|
||||
raise ValueError()
|
||||
for key in response.keys():
|
||||
if key not in [
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key]:
|
||||
raise ValueError()
|
||||
extra_data = response_msg[curr_pos:]
|
||||
break
|
||||
except ValueError:
|
||||
response = None
|
||||
if response is None:
|
||||
extra_data = response_msg
|
||||
return response, extra_data
|
||||
|
||||
|
||||
class BlobRequest:
|
||||
def __init__(self, requests: typing.List[blob_request_types]) -> None:
|
||||
self.requests = requests
|
||||
|
||||
def to_dict(self):
|
||||
d = {}
|
||||
for request in self.requests:
|
||||
d.update(request.to_dict())
|
||||
return d
|
||||
|
||||
def _get_request(self, request_type: blob_request_types):
|
||||
request = tuple(filter(lambda r: type(r) == request_type, self.requests))
|
||||
if request:
|
||||
return request[0]
|
||||
|
||||
def get_availability_request(self) -> typing.Optional[BlobAvailabilityRequest]:
|
||||
response = self._get_request(BlobAvailabilityRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_price_request(self) -> typing.Optional[BlobPriceRequest]:
|
||||
response = self._get_request(BlobPriceRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_blob_request(self) -> typing.Optional[BlobDownloadRequest]:
|
||||
response = self._get_request(BlobDownloadRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_address_request(self) -> typing.Optional[BlobPaymentAddressRequest]:
|
||||
response = self._get_request(BlobPaymentAddressRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def serialize(self) -> bytes:
|
||||
return json.dumps(self.to_dict()).encode()
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, data: bytes) -> 'BlobRequest':
|
||||
request = json.loads(data)
|
||||
return cls([
|
||||
request_type(**request)
|
||||
for request_type in (BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
|
||||
BlobPaymentAddressRequest)
|
||||
if request_type.key in request
|
||||
])
|
||||
|
||||
@classmethod
|
||||
def make_request_for_blob_hash(cls, blob_hash: str) -> 'BlobRequest':
|
||||
return cls(
|
||||
[BlobAvailabilityRequest([blob_hash]), BlobPriceRequest(0.0), BlobDownloadRequest(blob_hash)]
|
||||
)
|
||||
|
||||
|
||||
class BlobResponse:
|
||||
def __init__(self, responses: typing.List[blob_response_types], blob_data: typing.Optional[bytes] = None) -> None:
|
||||
self.responses = responses
|
||||
self.blob_data = blob_data
|
||||
|
||||
def to_dict(self):
|
||||
d = {}
|
||||
for response in self.responses:
|
||||
d.update(response.to_dict())
|
||||
return d
|
||||
|
||||
def _get_response(self, response_type: blob_response_types):
|
||||
response = tuple(filter(lambda r: type(r) == response_type, self.responses))
|
||||
if response:
|
||||
return response[0]
|
||||
|
||||
def get_error_response(self) -> typing.Optional[BlobErrorResponse]:
|
||||
error = self._get_response(BlobErrorResponse)
|
||||
if error:
|
||||
log.error(error)
|
||||
return error
|
||||
|
||||
def get_availability_response(self) -> typing.Optional[BlobAvailabilityResponse]:
|
||||
response = self._get_response(BlobAvailabilityResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_price_response(self) -> typing.Optional[BlobPriceResponse]:
|
||||
response = self._get_response(BlobPriceResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_blob_response(self) -> typing.Optional[BlobDownloadResponse]:
|
||||
response = self._get_response(BlobDownloadResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_address_response(self) -> typing.Optional[BlobPaymentAddressResponse]:
|
||||
response = self._get_response(BlobPaymentAddressResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def serialize(self) -> bytes:
|
||||
return json.dumps(self.to_dict()).encode()
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, data: bytes) -> 'BlobResponse':
|
||||
response, extra = _parse_blob_response(data)
|
||||
requests = []
|
||||
if response:
|
||||
requests.extend([
|
||||
response_type(**response)
|
||||
for response_type in (BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
|
||||
BlobErrorResponse, BlobPaymentAddressResponse)
|
||||
if response_type.key in response
|
||||
])
|
||||
return cls(requests, extra)
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
from json.decoder import JSONDecodeError
|
||||
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
|
||||
from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPriceResponse, BlobDownloadResponse, \
|
||||
BlobPaymentAddressResponse
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.blob.blob_manager import BlobFileManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.buf = b''
|
||||
self.transport = None
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
|
||||
def send_response(self, responses: typing.List[blob_response_types]):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
self.transport.write(BlobResponse(to_send).serialize())
|
||||
|
||||
async def handle_request(self, request: BlobRequest):
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
|
||||
responses = []
|
||||
address_request = request.get_address_request()
|
||||
if address_request:
|
||||
responses.append(BlobPaymentAddressResponse(lbrycrd_address=self.lbrycrd_address))
|
||||
availability_request = request.get_availability_request()
|
||||
if availability_request:
|
||||
responses.append(BlobAvailabilityResponse(available_blobs=list(set((
|
||||
filter(lambda blob_hash: blob_hash in self.blob_manager.completed_blob_hashes,
|
||||
availability_request.requested_blobs)
|
||||
)))))
|
||||
price_request = request.get_price_request()
|
||||
if price_request:
|
||||
responses.append(BlobPriceResponse(blob_data_payment_rate='RATE_ACCEPTED'))
|
||||
download_request = request.get_blob_request()
|
||||
|
||||
if download_request:
|
||||
blob = self.blob_manager.get_blob(download_request.requested_blob)
|
||||
if blob.get_is_verified():
|
||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||
self.send_response(responses)
|
||||
log.info("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
sent = await blob.sendfile(self)
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
|
||||
if responses:
|
||||
self.send_response(responses)
|
||||
# self.transport.close()
|
||||
|
||||
def data_received(self, data):
|
||||
request = None
|
||||
if data:
|
||||
message, separator, remainder = data.rpartition(b'}')
|
||||
if not separator:
|
||||
self.buf += data
|
||||
return
|
||||
try:
|
||||
request = BlobRequest.deserialize(data)
|
||||
self.buf = remainder
|
||||
except JSONDecodeError:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
log.error("failed to decode blob request from %s:%i (%i bytes): %s", peer_address, peer_port,
|
||||
len(data), '' if not data else binascii.hexlify(data).decode())
|
||||
if not request:
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
log.warning("failed to decode blob request from %s:%i", peer_address, peer_port)
|
||||
self.transport.close()
|
||||
return
|
||||
self.loop.create_task(self.handle_request(request))
|
||||
|
||||
|
||||
class BlobServer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
raise Exception("already running")
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: BlobServerProtocol(self.loop, self.blob_manager, self.lbrycrd_address),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
log.info("Blob server listening on TCP %s:%i", interface, port)
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
|
||||
self.server_task = self.loop.create_task(_start_server())
|
||||
|
||||
def stop_server(self):
|
||||
if self.server_task:
|
||||
self.server_task.cancel()
|
||||
self.server_task = None
|
||||
log.info("Stopped blob server")
|
||||
@@ -1,4 +1,8 @@
|
||||
import hashlib
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
|
||||
|
||||
backend = default_backend()
|
||||
|
||||
|
||||
def get_lbry_hash_obj():
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import inspect
|
||||
import logging
|
||||
TRACE = 5
|
||||
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
#
|
||||
# _srcfile is used when walking the stack to check when we've got the first
|
||||
# caller stack frame.
|
||||
#
|
||||
if hasattr(sys, 'frozen'): # support for py2exe
|
||||
_srcfile = "logging{}__init__{}".format(os.sep, __file__[-4:])
|
||||
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
|
||||
_srcfile = __file__[:-4] + '.py'
|
||||
else:
|
||||
_srcfile = __file__
|
||||
_srcfile = os.path.normcase(_srcfile)
|
||||
|
||||
|
||||
def findCaller(srcfile=None):
|
||||
"""Returns the filename, line number and function name of the caller"""
|
||||
srcfile = srcfile or _srcfile
|
||||
f = inspect.currentframe()
|
||||
# On some versions of IronPython, currentframe() returns None if
|
||||
# IronPython isn't run with -X:Frames.
|
||||
if f is not None:
|
||||
f = f.f_back
|
||||
rv = "(unknown file)", 0, "(unknown function)"
|
||||
while hasattr(f, "f_code"):
|
||||
co = f.f_code
|
||||
filename = os.path.normcase(co.co_filename)
|
||||
# ignore any function calls that are in this file
|
||||
if filename == srcfile:
|
||||
f = f.f_back
|
||||
continue
|
||||
rv = (filename, f.f_lineno, co.co_name)
|
||||
break
|
||||
return rv
|
||||
|
||||
|
||||
###
|
||||
|
||||
class Logger(logging.Logger):
|
||||
"""A logger that has an extra `fail` method useful for handling twisted failures."""
|
||||
|
||||
def fail(self, callback=None, *args, **kwargs):
|
||||
"""Returns a function to log a failure from an errback.
|
||||
|
||||
The returned function appends the error message and extracts
|
||||
the traceback from `err`.
|
||||
|
||||
Example usage:
|
||||
d.addErrback(log.fail(), 'This is an error message')
|
||||
|
||||
Although odd, making the method call is necessary to extract
|
||||
out useful filename and line number information; otherwise the
|
||||
reported values are from inside twisted's deferred handling
|
||||
code.
|
||||
|
||||
Args:
|
||||
callback: callable to call after making the log. The first argument
|
||||
will be the `err` from the deferred
|
||||
args: extra arguments to pass into `callback`
|
||||
|
||||
Returns: a function that takes the following arguments:
|
||||
err: twisted.python.failure.Failure
|
||||
msg: the message to log, using normal logging string iterpolation.
|
||||
msg_args: the values to substitute into `msg`
|
||||
msg_kwargs: set `level` to change from the default ERROR severity. Other
|
||||
keywoards are treated as normal log kwargs.
|
||||
"""
|
||||
fn, lno, func = findCaller()
|
||||
|
||||
def _fail(err, msg, *msg_args, **msg_kwargs):
|
||||
level = msg_kwargs.pop('level', logging.ERROR)
|
||||
msg += ": %s"
|
||||
msg_args += (err.getErrorMessage(),)
|
||||
exc_info = (err.type, err.value, err.getTracebackObject())
|
||||
record = self.makeRecord(
|
||||
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
|
||||
self.handle(record)
|
||||
if callback:
|
||||
try:
|
||||
return callback(err, *args, **kwargs)
|
||||
except Exception:
|
||||
# log.fail is almost always called within an
|
||||
# errback. If callback fails and we didn't catch
|
||||
# the exception we would need to attach a second
|
||||
# errback to deal with that, which we will almost
|
||||
# never do and then we end up with an unhandled
|
||||
# error that will get swallowed by twisted
|
||||
self.exception('Failed to run callback')
|
||||
|
||||
return _fail
|
||||
|
||||
def trace(self, msg, *args, **kwargs):
|
||||
if self.isEnabledFor(TRACE):
|
||||
self._log(TRACE, msg, args, **kwargs)
|
||||
|
||||
|
||||
def install_logger():
|
||||
current = logging.getLoggerClass()
|
||||
if current is not Logger:
|
||||
logging.setLoggerClass(Logger)
|
||||
logging.addLevelName(TRACE, 'TRACE')
|
||||
@@ -1,7 +0,0 @@
|
||||
Francois Aucamp <faucamp@csir.co.za>
|
||||
|
||||
Thanks goes to the following people for providing patches/suggestions/tests:
|
||||
|
||||
Neil Kleynhans <ntkleynhans@csir.co.za>
|
||||
Haiyang Ma <haiyang.ma@maidsafe.net>
|
||||
Bryan McAlister <bmcalister@csir.co.za>
|
||||
@@ -1,165 +0,0 @@
|
||||
GNU LESSER GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
|
||||
This version of the GNU Lesser General Public License incorporates
|
||||
the terms and conditions of version 3 of the GNU General Public
|
||||
License, supplemented by the additional permissions listed below.
|
||||
|
||||
0. Additional Definitions.
|
||||
|
||||
As used herein, "this License" refers to version 3 of the GNU Lesser
|
||||
General Public License, and the "GNU GPL" refers to version 3 of the GNU
|
||||
General Public License.
|
||||
|
||||
"The Library" refers to a covered work governed by this License,
|
||||
other than an Application or a Combined Work as defined below.
|
||||
|
||||
An "Application" is any work that makes use of an interface provided
|
||||
by the Library, but which is not otherwise based on the Library.
|
||||
Defining a subclass of a class defined by the Library is deemed a mode
|
||||
of using an interface provided by the Library.
|
||||
|
||||
A "Combined Work" is a work produced by combining or linking an
|
||||
Application with the Library. The particular version of the Library
|
||||
with which the Combined Work was made is also called the "Linked
|
||||
Version".
|
||||
|
||||
The "Minimal Corresponding Source" for a Combined Work means the
|
||||
Corresponding Source for the Combined Work, excluding any source code
|
||||
for portions of the Combined Work that, considered in isolation, are
|
||||
based on the Application, and not on the Linked Version.
|
||||
|
||||
The "Corresponding Application Code" for a Combined Work means the
|
||||
object code and/or source code for the Application, including any data
|
||||
and utility programs needed for reproducing the Combined Work from the
|
||||
Application, but excluding the System Libraries of the Combined Work.
|
||||
|
||||
1. Exception to Section 3 of the GNU GPL.
|
||||
|
||||
You may convey a covered work under sections 3 and 4 of this License
|
||||
without being bound by section 3 of the GNU GPL.
|
||||
|
||||
2. Conveying Modified Versions.
|
||||
|
||||
If you modify a copy of the Library, and, in your modifications, a
|
||||
facility refers to a function or data to be supplied by an Application
|
||||
that uses the facility (other than as an argument passed when the
|
||||
facility is invoked), then you may convey a copy of the modified
|
||||
version:
|
||||
|
||||
a) under this License, provided that you make a good faith effort to
|
||||
ensure that, in the event an Application does not supply the
|
||||
function or data, the facility still operates, and performs
|
||||
whatever part of its purpose remains meaningful, or
|
||||
|
||||
b) under the GNU GPL, with none of the additional permissions of
|
||||
this License applicable to that copy.
|
||||
|
||||
3. Object Code Incorporating Material from Library Header Files.
|
||||
|
||||
The object code form of an Application may incorporate material from
|
||||
a header file that is part of the Library. You may convey such object
|
||||
code under terms of your choice, provided that, if the incorporated
|
||||
material is not limited to numerical parameters, data structure
|
||||
layouts and accessors, or small macros, inline functions and templates
|
||||
(ten or fewer lines in length), you do both of the following:
|
||||
|
||||
a) Give prominent notice with each copy of the object code that the
|
||||
Library is used in it and that the Library and its use are
|
||||
covered by this License.
|
||||
|
||||
b) Accompany the object code with a copy of the GNU GPL and this license
|
||||
document.
|
||||
|
||||
4. Combined Works.
|
||||
|
||||
You may convey a Combined Work under terms of your choice that,
|
||||
taken together, effectively do not restrict modification of the
|
||||
portions of the Library contained in the Combined Work and reverse
|
||||
engineering for debugging such modifications, if you also do each of
|
||||
the following:
|
||||
|
||||
a) Give prominent notice with each copy of the Combined Work that
|
||||
the Library is used in it and that the Library and its use are
|
||||
covered by this License.
|
||||
|
||||
b) Accompany the Combined Work with a copy of the GNU GPL and this license
|
||||
document.
|
||||
|
||||
c) For a Combined Work that displays copyright notices during
|
||||
execution, include the copyright notice for the Library among
|
||||
these notices, as well as a reference directing the user to the
|
||||
copies of the GNU GPL and this license document.
|
||||
|
||||
d) Do one of the following:
|
||||
|
||||
0) Convey the Minimal Corresponding Source under the terms of this
|
||||
License, and the Corresponding Application Code in a form
|
||||
suitable for, and under terms that permit, the user to
|
||||
recombine or relink the Application with a modified version of
|
||||
the Linked Version to produce a modified Combined Work, in the
|
||||
manner specified by section 6 of the GNU GPL for conveying
|
||||
Corresponding Source.
|
||||
|
||||
1) Use a suitable shared library mechanism for linking with the
|
||||
Library. A suitable mechanism is one that (a) uses at run time
|
||||
a copy of the Library already present on the user's computer
|
||||
system, and (b) will operate properly with a modified version
|
||||
of the Library that is interface-compatible with the Linked
|
||||
Version.
|
||||
|
||||
e) Provide Installation Information, but only if you would otherwise
|
||||
be required to provide such information under section 6 of the
|
||||
GNU GPL, and only to the extent that such information is
|
||||
necessary to install and execute a modified version of the
|
||||
Combined Work produced by recombining or relinking the
|
||||
Application with a modified version of the Linked Version. (If
|
||||
you use option 4d0, the Installation Information must accompany
|
||||
the Minimal Corresponding Source and Corresponding Application
|
||||
Code. If you use option 4d1, you must provide the Installation
|
||||
Information in the manner specified by section 6 of the GNU GPL
|
||||
for conveying Corresponding Source.)
|
||||
|
||||
5. Combined Libraries.
|
||||
|
||||
You may place library facilities that are a work based on the
|
||||
Library side by side in a single library together with other library
|
||||
facilities that are not Applications and are not covered by this
|
||||
License, and convey such a combined library under terms of your
|
||||
choice, if you do both of the following:
|
||||
|
||||
a) Accompany the combined library with a copy of the same work based
|
||||
on the Library, uncombined with any other library facilities,
|
||||
conveyed under the terms of this License.
|
||||
|
||||
b) Give prominent notice with the combined library that part of it
|
||||
is a work based on the Library, and explaining where to find the
|
||||
accompanying uncombined form of the same work.
|
||||
|
||||
6. Revised Versions of the GNU Lesser General Public License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions
|
||||
of the GNU Lesser General Public License from time to time. Such new
|
||||
versions will be similar in spirit to the present version, but may
|
||||
differ in detail to address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Library as you received it specifies that a certain numbered version
|
||||
of the GNU Lesser General Public License "or any later version"
|
||||
applies to it, you have the option of following the terms and
|
||||
conditions either of that published version or of any later version
|
||||
published by the Free Software Foundation. If the Library as you
|
||||
received it does not specify a version number of the GNU Lesser
|
||||
General Public License, you may choose any version of the GNU Lesser
|
||||
General Public License ever published by the Free Software Foundation.
|
||||
|
||||
If the Library as you received it specifies that a proxy can decide
|
||||
whether future versions of the GNU Lesser General Public License shall
|
||||
apply, that proxy's public statement of acceptance of any version is
|
||||
permanent authorization for you to choose that version for the
|
||||
Library.
|
||||
@@ -0,0 +1,65 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobAnnouncer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, node: 'Node', storage: 'SQLiteStorage'):
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
self.storage = storage
|
||||
self.pending_call: asyncio.Handle = None
|
||||
self.announce_task: asyncio.Task = None
|
||||
self.running = False
|
||||
self.announce_queue: typing.List[str] = []
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
blob_hashes = await self.storage.get_blobs_to_announce()
|
||||
if blob_hashes:
|
||||
self.announce_queue.extend(blob_hashes)
|
||||
log.info("%i blobs to announce", len(blob_hashes))
|
||||
batch = []
|
||||
while len(self.announce_queue):
|
||||
cnt = 0
|
||||
announced = []
|
||||
while self.announce_queue and cnt < batch_size:
|
||||
blob_hash = self.announce_queue.pop()
|
||||
announced.append(blob_hash)
|
||||
batch.append(self.node.announce_blob(blob_hash))
|
||||
cnt += 1
|
||||
to_await = []
|
||||
while batch:
|
||||
to_await.append(batch.pop())
|
||||
if to_await:
|
||||
await asyncio.gather(*tuple(to_await), loop=self.loop)
|
||||
await self.storage.update_last_announced_blobs(announced, self.loop.time())
|
||||
log.info("announced %i blobs", len(announced))
|
||||
if self.running:
|
||||
self.pending_call = self.loop.call_later(60, self.announce, batch_size)
|
||||
|
||||
def announce(self, batch_size: typing.Optional[int] = 10):
|
||||
self.announce_task = self.loop.create_task(self._announce(batch_size))
|
||||
|
||||
def start(self, batch_size: typing.Optional[int] = 10):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self.announce(batch_size)
|
||||
|
||||
def stop(self):
|
||||
self.running = False
|
||||
if self.pending_call:
|
||||
if not self.pending_call.cancelled():
|
||||
self.pending_call.cancel()
|
||||
self.pending_call = None
|
||||
if self.announce_task:
|
||||
if not (self.announce_task.done() or self.announce_task.cancelled()):
|
||||
self.announce_task.cancel()
|
||||
self.announce_task = None
|
||||
@@ -1,81 +0,0 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger()
|
||||
|
||||
MIN_DELAY = 0.0
|
||||
MAX_DELAY = 0.01
|
||||
DELAY_INCREMENT = 0.0001
|
||||
QUEUE_SIZE_THRESHOLD = 100
|
||||
|
||||
|
||||
class CallLaterManager:
|
||||
def __init__(self, callLater):
|
||||
"""
|
||||
:param callLater: (IReactorTime.callLater)
|
||||
"""
|
||||
|
||||
self._callLater = callLater
|
||||
self._pendingCallLaters = []
|
||||
self._delay = MIN_DELAY
|
||||
|
||||
def get_min_delay(self):
|
||||
self._pendingCallLaters = [cl for cl in self._pendingCallLaters if cl.active()]
|
||||
queue_size = len(self._pendingCallLaters)
|
||||
if queue_size > QUEUE_SIZE_THRESHOLD:
|
||||
self._delay = min((self._delay + DELAY_INCREMENT), MAX_DELAY)
|
||||
else:
|
||||
self._delay = max((self._delay - 2.0 * DELAY_INCREMENT), MIN_DELAY)
|
||||
return self._delay
|
||||
|
||||
def _cancel(self, call_later):
|
||||
"""
|
||||
:param call_later: DelayedCall
|
||||
:return: (callable) canceller function
|
||||
"""
|
||||
|
||||
def cancel(reason=None):
|
||||
"""
|
||||
:param reason: reason for cancellation, this is returned after cancelling the DelayedCall
|
||||
:return: reason
|
||||
"""
|
||||
|
||||
if call_later.active():
|
||||
call_later.cancel()
|
||||
if call_later in self._pendingCallLaters:
|
||||
self._pendingCallLaters.remove(call_later)
|
||||
return reason
|
||||
return cancel
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
Cancel any callLaters that are still running
|
||||
"""
|
||||
|
||||
from twisted.internet import defer
|
||||
while self._pendingCallLaters:
|
||||
canceller = self._cancel(self._pendingCallLaters[0])
|
||||
try:
|
||||
canceller()
|
||||
except (defer.CancelledError, defer.AlreadyCalledError, ValueError):
|
||||
pass
|
||||
|
||||
def call_later(self, when, what, *args, **kwargs):
|
||||
"""
|
||||
Schedule a call later and get a canceller callback function
|
||||
|
||||
:param when: (float) delay in seconds
|
||||
:param what: (callable)
|
||||
:param args: (*tuple) args to be passed to the callable
|
||||
:param kwargs: (**dict) kwargs to be passed to the callable
|
||||
|
||||
:return: (tuple) twisted.internet.base.DelayedCall object, canceller function
|
||||
"""
|
||||
|
||||
call_later = self._callLater(when, what, *args, **kwargs)
|
||||
canceller = self._cancel(call_later)
|
||||
self._pendingCallLaters.append(call_later)
|
||||
return call_later, canceller
|
||||
|
||||
def call_soon(self, what, *args, **kwargs):
|
||||
delay = self.get_min_delay()
|
||||
return self.call_later(delay, what, *args, **kwargs)
|
||||
@@ -1,61 +1,40 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# This library is free software, distributed under the terms of
|
||||
# the GNU Lesser General Public License Version 3, or any later version.
|
||||
# See the COPYING file included in this archive
|
||||
#
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
""" This module defines the charaterizing constants of the Kademlia network
|
||||
|
||||
C{checkRefreshInterval} and C{udpDatagramMaxSize} are implementation-specific
|
||||
constants, and do not affect general Kademlia operation.
|
||||
"""
|
||||
|
||||
######### KADEMLIA CONSTANTS ###########
|
||||
|
||||
#: Small number Representing the degree of parallelism in network calls
|
||||
alpha = 3
|
||||
|
||||
#: Maximum number of contacts stored in a bucket; this should be an even number
|
||||
hash_class = hashlib.sha384
|
||||
hash_length = hash_class().digest_size
|
||||
hash_bits = hash_length * 8
|
||||
alpha = 5
|
||||
k = 8
|
||||
|
||||
#: Maximum number of contacts stored in the replacement cache
|
||||
replacementCacheSize = 8
|
||||
|
||||
#: Timeout for network operations (in seconds)
|
||||
rpcTimeout = 5
|
||||
|
||||
# number of rpc attempts to make before a timeout results in the node being removed as a contact
|
||||
rpcAttempts = 5
|
||||
# time window to count failures (in seconds)
|
||||
rpcAttemptsPruningTimeWindow = 600
|
||||
|
||||
# Delay between iterations of iterative node lookups (for loose parallelism) (in seconds)
|
||||
iterativeLookupDelay = rpcTimeout / 2
|
||||
|
||||
#: If a k-bucket has not been used for this amount of time, refresh it (in seconds)
|
||||
refreshTimeout = 3600 # 1 hour
|
||||
#: The interval at which nodes replicate (republish/refresh) data they are holding
|
||||
replicateInterval = refreshTimeout
|
||||
# The time it takes for data to expire in the network; the original publisher of the data
|
||||
# will also republish the data at this time if it is still valid
|
||||
dataExpireTimeout = 86400 # 24 hours
|
||||
|
||||
tokenSecretChangeInterval = 300 # 5 minutes
|
||||
|
||||
######## IMPLEMENTATION-SPECIFIC CONSTANTS ###########
|
||||
|
||||
#: The interval for the node to check whether any buckets need refreshing
|
||||
checkRefreshInterval = refreshTimeout / 5
|
||||
|
||||
#: Max size of a single UDP datagram, in bytes. If a message is larger than this, it will
|
||||
#: be spread across several UDP packets.
|
||||
udpDatagramMaxSize = 8192 # 8 KB
|
||||
|
||||
key_bits = 384
|
||||
|
||||
replacement_cache_size = 8
|
||||
rpc_timeout = 5.0
|
||||
rpc_attempts = 5
|
||||
rpc_attempts_pruning_window = 600
|
||||
iterative_lookup_delay = rpc_timeout / 2.0 # TODO: use config val / 2 if rpc timeout is provided
|
||||
refresh_interval = 3600 # 1 hour
|
||||
replicate_interval = refresh_interval
|
||||
data_expiration = 86400 # 24 hours
|
||||
token_secret_refresh_interval = 300 # 5 minutes
|
||||
check_refresh_interval = refresh_interval / 5
|
||||
max_datagram_size = 8192 # 8 KB
|
||||
rpc_id_length = 20
|
||||
protocol_version = 1
|
||||
bottom_out_limit = 3
|
||||
msg_size_limit = max_datagram_size - 26
|
||||
|
||||
protocolVersion = 1
|
||||
|
||||
def digest(data: bytes) -> bytes:
|
||||
h = hash_class()
|
||||
h.update(data)
|
||||
return h.digest()
|
||||
|
||||
|
||||
def generate_id(num=None) -> bytes:
|
||||
if num is not None:
|
||||
return digest(str(num).encode())
|
||||
else:
|
||||
return digest(os.urandom(32))
|
||||
|
||||
|
||||
def generate_rpc_id(num=None) -> bytes:
|
||||
return generate_id(num)[:rpc_id_length]
|
||||
@@ -1,189 +0,0 @@
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from functools import reduce
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
def is_valid_ipv4(address):
|
||||
try:
|
||||
ip = ipaddress.ip_address(address)
|
||||
return ip.version == 4
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
|
||||
|
||||
class _Contact:
|
||||
""" Encapsulation for remote contact
|
||||
|
||||
This class contains information on a single remote contact, and also
|
||||
provides a direct RPC API to the remote node which it represents
|
||||
"""
|
||||
|
||||
def __init__(self, contactManager, id, ipAddress, udpPort, networkProtocol, firstComm):
|
||||
if id is not None:
|
||||
if not len(id) == constants.key_bits // 8:
|
||||
raise ValueError("invalid node id: {}".format(hexlify(id).decode()))
|
||||
if not 0 <= udpPort <= 65536:
|
||||
raise ValueError("invalid port")
|
||||
if not is_valid_ipv4(ipAddress):
|
||||
raise ValueError("invalid ip address")
|
||||
self._contactManager = contactManager
|
||||
self._id = id
|
||||
self.address = ipAddress
|
||||
self.port = udpPort
|
||||
self._networkProtocol = networkProtocol
|
||||
self.commTime = firstComm
|
||||
self.getTime = self._contactManager._get_time
|
||||
self.lastReplied = None
|
||||
self.lastRequested = None
|
||||
self.protocolVersion = 0
|
||||
self._token = (None, 0) # token, timestamp
|
||||
|
||||
def update_token(self, token):
|
||||
self._token = token, self.getTime() if token else 0
|
||||
|
||||
@property
|
||||
def token(self):
|
||||
# expire the token 1 minute early to be safe
|
||||
return self._token[0] if self._token[1] + 240 > self.getTime() else None
|
||||
|
||||
@property
|
||||
def lastInteracted(self):
|
||||
return max(self.lastRequested or 0, self.lastReplied or 0, self.lastFailed or 0)
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self._id
|
||||
|
||||
def log_id(self, short=True):
|
||||
if not self.id:
|
||||
return "not initialized"
|
||||
id_hex = hexlify(self.id)
|
||||
return id_hex if not short else id_hex[:8]
|
||||
|
||||
@property
|
||||
def failedRPCs(self):
|
||||
return len(self.failures)
|
||||
|
||||
@property
|
||||
def lastFailed(self):
|
||||
return (self.failures or [None])[-1]
|
||||
|
||||
@property
|
||||
def failures(self):
|
||||
return self._contactManager._rpc_failures.get((self.address, self.port), [])
|
||||
|
||||
@property
|
||||
def contact_is_good(self):
|
||||
"""
|
||||
:return: False if contact is bad, None if contact is unknown, or True if contact is good
|
||||
"""
|
||||
failures = self.failures
|
||||
now = self.getTime()
|
||||
delay = constants.checkRefreshInterval
|
||||
|
||||
if failures:
|
||||
if self.lastReplied and len(failures) >= 2 and self.lastReplied < failures[-2]:
|
||||
return False
|
||||
elif self.lastReplied and len(failures) >= 2 and self.lastReplied > failures[-2]:
|
||||
pass # handled below
|
||||
elif len(failures) >= 2:
|
||||
return False
|
||||
|
||||
if self.lastReplied and self.lastReplied > now - delay:
|
||||
return True
|
||||
if self.lastReplied and self.lastRequested and self.lastRequested > now - delay:
|
||||
return True
|
||||
return None
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, _Contact):
|
||||
raise TypeError("invalid type to compare with Contact: %s" % str(type(other)))
|
||||
return (self.id, self.address, self.port) == (other.id, other.address, other.port)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.id, self.address, self.port))
|
||||
|
||||
def compact_ip(self):
|
||||
compact_ip = reduce(
|
||||
lambda buff, x: buff + bytearray([int(x)]), self.address.split('.'), bytearray())
|
||||
return compact_ip
|
||||
|
||||
def set_id(self, id):
|
||||
if not self._id:
|
||||
self._id = id
|
||||
|
||||
def update_last_replied(self):
|
||||
self.lastReplied = int(self.getTime())
|
||||
|
||||
def update_last_requested(self):
|
||||
self.lastRequested = int(self.getTime())
|
||||
|
||||
def update_last_failed(self):
|
||||
failures = self._contactManager._rpc_failures.get((self.address, self.port), [])
|
||||
failures.append(self.getTime())
|
||||
self._contactManager._rpc_failures[(self.address, self.port)] = failures
|
||||
|
||||
def update_protocol_version(self, version):
|
||||
self.protocolVersion = version
|
||||
|
||||
def __str__(self):
|
||||
return '<%s.%s object; IP address: %s, UDP port: %d>' % (
|
||||
self.__module__, self.__class__.__name__, self.address, self.port)
|
||||
|
||||
def __getattr__(self, name):
|
||||
""" This override allows the host node to call a method of the remote
|
||||
node (i.e. this contact) as if it was a local function.
|
||||
|
||||
For instance, if C{remoteNode} is a instance of C{Contact}, the
|
||||
following will result in C{remoteNode}'s C{test()} method to be
|
||||
called with argument C{123}::
|
||||
remoteNode.test(123)
|
||||
|
||||
Such a RPC method call will return a Deferred, which will callback
|
||||
when the contact responds with the result (or an error occurs).
|
||||
This happens via this contact's C{_networkProtocol} object (i.e. the
|
||||
host Node's C{_protocol} object).
|
||||
"""
|
||||
|
||||
if name not in ['ping', 'findValue', 'findNode', 'store']:
|
||||
raise AttributeError("unknown command: %s" % name)
|
||||
|
||||
def _sendRPC(*args, **kwargs):
|
||||
return self._networkProtocol.sendRPC(self, name.encode(), args)
|
||||
|
||||
return _sendRPC
|
||||
|
||||
|
||||
class ContactManager:
|
||||
def __init__(self, get_time=None):
|
||||
if not get_time:
|
||||
from twisted.internet import reactor
|
||||
get_time = reactor.seconds
|
||||
self._get_time = get_time
|
||||
self._contacts = {}
|
||||
self._rpc_failures = {}
|
||||
|
||||
def get_contact(self, id, address, port):
|
||||
for contact in self._contacts.values():
|
||||
if contact.id == id and contact.address == address and contact.port == port:
|
||||
return contact
|
||||
|
||||
def make_contact(self, id, ipAddress, udpPort, networkProtocol, firstComm=0):
|
||||
contact = self.get_contact(id, ipAddress, udpPort)
|
||||
if contact:
|
||||
return contact
|
||||
contact = _Contact(self, id, ipAddress, udpPort, networkProtocol, firstComm or self._get_time())
|
||||
self._contacts[(id, ipAddress, udpPort)] = contact
|
||||
return contact
|
||||
|
||||
def is_ignored(self, origin_tuple):
|
||||
failed_rpc_count = len(self._prune_failures(origin_tuple))
|
||||
return failed_rpc_count > constants.rpcAttempts
|
||||
|
||||
def _prune_failures(self, origin_tuple):
|
||||
# Prunes recorded failures to the last time window of attempts
|
||||
pruning_limit = self._get_time() - constants.rpcAttemptsPruningTimeWindow
|
||||
pruned = list(filter(lambda t: t >= pruning_limit, self._rpc_failures.get(origin_tuple, [])))
|
||||
self._rpc_failures[origin_tuple] = pruned
|
||||
return pruned
|
||||
@@ -1,66 +0,0 @@
|
||||
from collections import UserDict
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class DictDataStore(UserDict):
|
||||
""" A datastore using an in-memory Python dictionary """
|
||||
#implements(IDataStore)
|
||||
|
||||
def __init__(self, getTime=None):
|
||||
# Dictionary format:
|
||||
# { <key>: (<contact>, <value>, <lastPublished>, <originallyPublished> <originalPublisherID>) }
|
||||
super().__init__()
|
||||
if not getTime:
|
||||
from twisted.internet import reactor
|
||||
getTime = reactor.seconds
|
||||
self._getTime = getTime
|
||||
self.completed_blobs = set()
|
||||
|
||||
def filter_bad_and_expired_peers(self, key):
|
||||
"""
|
||||
Returns only non-expired and unknown/good peers
|
||||
"""
|
||||
return filter(
|
||||
lambda peer:
|
||||
self._getTime() - peer[3] < constants.dataExpireTimeout and peer[0].contact_is_good is not False,
|
||||
self[key]
|
||||
)
|
||||
|
||||
def filter_expired_peers(self, key):
|
||||
"""
|
||||
Returns only non-expired peers
|
||||
"""
|
||||
return filter(lambda peer: self._getTime() - peer[3] < constants.dataExpireTimeout, self[key])
|
||||
|
||||
def removeExpiredPeers(self):
|
||||
expired_keys = []
|
||||
for key in self.keys():
|
||||
unexpired_peers = list(self.filter_expired_peers(key))
|
||||
if not unexpired_peers:
|
||||
expired_keys.append(key)
|
||||
else:
|
||||
self[key] = unexpired_peers
|
||||
for key in expired_keys:
|
||||
del self[key]
|
||||
|
||||
def hasPeersForBlob(self, key):
|
||||
return True if key in self and len(tuple(self.filter_bad_and_expired_peers(key))) else False
|
||||
|
||||
def addPeerToBlob(self, contact, key, compact_address, lastPublished, originallyPublished, originalPublisherID):
|
||||
if key in self:
|
||||
if compact_address not in map(lambda store_tuple: store_tuple[1], self[key]):
|
||||
self[key].append(
|
||||
(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)
|
||||
)
|
||||
else:
|
||||
self[key] = [(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)]
|
||||
|
||||
def getPeersForBlob(self, key):
|
||||
return [] if key not in self else [val[1] for val in self.filter_bad_and_expired_peers(key)]
|
||||
|
||||
def getStoringContacts(self):
|
||||
contacts = set()
|
||||
for key in self:
|
||||
for values in self[key]:
|
||||
contacts.add(values[0])
|
||||
return list(contacts)
|
||||
@@ -1,45 +1,23 @@
|
||||
import binascii
|
||||
#import exceptions
|
||||
|
||||
# this is a dict of {"exceptions.<exception class name>": exception class} items used to raise
|
||||
# remote built-in exceptions locally
|
||||
BUILTIN_EXCEPTIONS = {
|
||||
# "exceptions.%s" % e: getattr(exceptions, e) for e in dir(exceptions) if not e.startswith("_")
|
||||
}
|
||||
class BaseKademliaException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DecodeError(Exception):
|
||||
class DecodeError(BaseKademliaException):
|
||||
"""
|
||||
Should be raised by an C{Encoding} implementation if decode operation
|
||||
fails
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class BucketFull(Exception):
|
||||
class BucketFull(BaseKademliaException):
|
||||
"""
|
||||
Raised when the bucket is full
|
||||
"""
|
||||
|
||||
|
||||
class RemoteException(BaseKademliaException):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownRemoteException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class TimeoutError(Exception):
|
||||
""" Raised when a RPC times out """
|
||||
|
||||
def __init__(self, remote_contact_id):
|
||||
# remote_contact_id is a binary blob so we need to convert it
|
||||
# into something more readable
|
||||
if remote_contact_id:
|
||||
msg = 'Timeout connecting to {}'.format(binascii.hexlify(remote_contact_id))
|
||||
else:
|
||||
msg = 'Timeout connecting to uninitialized node'
|
||||
super().__init__(msg)
|
||||
self.remote_contact_id = remote_contact_id
|
||||
|
||||
|
||||
class TransportNotConnected(Exception):
|
||||
class TransportNotConnected(BaseKademliaException):
|
||||
pass
|
||||
@@ -1,117 +0,0 @@
|
||||
from zope.interface import Interface
|
||||
|
||||
|
||||
class IDataStore(Interface):
|
||||
""" Interface for classes implementing physical storage (for data
|
||||
published via the "STORE" RPC) for the Kademlia DHT
|
||||
|
||||
@note: This provides an interface for a dict-like object
|
||||
"""
|
||||
|
||||
def keys(self):
|
||||
""" Return a list of the keys in this data store """
|
||||
pass
|
||||
|
||||
def removeExpiredPeers(self):
|
||||
pass
|
||||
|
||||
def hasPeersForBlob(self, key):
|
||||
pass
|
||||
|
||||
def addPeerToBlob(self, key, value, lastPublished, originallyPublished, originalPublisherID):
|
||||
pass
|
||||
|
||||
def getPeersForBlob(self, key):
|
||||
pass
|
||||
|
||||
def removePeer(self, key):
|
||||
pass
|
||||
|
||||
|
||||
class IRoutingTable(Interface):
|
||||
""" Interface for RPC message translators/formatters
|
||||
|
||||
Classes inheriting from this should provide a suitable routing table for
|
||||
a parent Node object (i.e. the local entity in the Kademlia network)
|
||||
"""
|
||||
|
||||
def __init__(self, parentNodeID):
|
||||
"""
|
||||
@param parentNodeID: The n-bit node ID of the node to which this
|
||||
routing table belongs
|
||||
@type parentNodeID: str
|
||||
"""
|
||||
|
||||
def addContact(self, contact):
|
||||
""" Add the given contact to the correct k-bucket; if it already
|
||||
exists, its status will be updated
|
||||
|
||||
@param contact: The contact to add to this node's k-buckets
|
||||
@type contact: kademlia.contact.Contact
|
||||
"""
|
||||
|
||||
def findCloseNodes(self, key, count, _rpcNodeID=None):
|
||||
""" Finds a number of known nodes closest to the node/value with the
|
||||
specified key.
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
@param count: the amount of contacts to return
|
||||
@type count: int
|
||||
@param _rpcNodeID: Used during RPC, this is be the sender's Node ID
|
||||
Whatever ID is passed in the parameter will get
|
||||
excluded from the list of returned contacts.
|
||||
@type _rpcNodeID: str
|
||||
|
||||
@return: A list of node contacts (C{kademlia.contact.Contact instances})
|
||||
closest to the specified key.
|
||||
This method will return C{k} (or C{count}, if specified)
|
||||
contacts if at all possible; it will only return fewer if the
|
||||
node is returning all of the contacts that it knows of.
|
||||
@rtype: list
|
||||
"""
|
||||
|
||||
def getContact(self, contactID):
|
||||
""" Returns the (known) contact with the specified node ID
|
||||
|
||||
@raise ValueError: No contact with the specified contact ID is known
|
||||
by this node
|
||||
"""
|
||||
|
||||
def getRefreshList(self, startIndex=0, force=False):
|
||||
""" Finds all k-buckets that need refreshing, starting at the
|
||||
k-bucket with the specified index, and returns IDs to be searched for
|
||||
in order to refresh those k-buckets
|
||||
|
||||
@param startIndex: The index of the bucket to start refreshing at;
|
||||
this bucket and those further away from it will
|
||||
be refreshed. For example, when joining the
|
||||
network, this node will set this to the index of
|
||||
the bucket after the one containing it's closest
|
||||
neighbour.
|
||||
@type startIndex: index
|
||||
@param force: If this is C{True}, all buckets (in the specified range)
|
||||
will be refreshed, regardless of the time they were last
|
||||
accessed.
|
||||
@type force: bool
|
||||
|
||||
@return: A list of node ID's that the parent node should search for
|
||||
in order to refresh the routing Table
|
||||
@rtype: list
|
||||
"""
|
||||
|
||||
def removeContact(self, contactID):
|
||||
""" Remove the contact with the specified node ID from the routing
|
||||
table
|
||||
|
||||
@param contactID: The node ID of the contact to remove
|
||||
@type contactID: str
|
||||
"""
|
||||
|
||||
def touchKBucket(self, key):
|
||||
""" Update the "last accessed" timestamp of the k-bucket which covers
|
||||
the range containing the specified key in the key/ID space
|
||||
|
||||
@param key: A key in the range of the target k-bucket
|
||||
@type key: str
|
||||
"""
|
||||
@@ -1,230 +0,0 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from lbrynet.dht.distance import Distance
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht import constants
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_contact(contact_list, node_id, address, port):
|
||||
for contact in contact_list:
|
||||
if contact.id == node_id and contact.address == address and contact.port == port:
|
||||
return contact
|
||||
raise IndexError(node_id)
|
||||
|
||||
|
||||
def expand_peer(compact_peer_info):
|
||||
host = "{}.{}.{}.{}".format(*compact_peer_info[:4])
|
||||
port = int.from_bytes(compact_peer_info[4:6], 'big')
|
||||
peer_node_id = compact_peer_info[6:]
|
||||
return (peer_node_id, host, port)
|
||||
|
||||
|
||||
class _IterativeFind:
|
||||
# TODO: use polymorphism to search for a value or node
|
||||
# instead of using a find_value flag
|
||||
def __init__(self, node, shortlist, key, rpc, exclude=None):
|
||||
self.exclude = set(exclude or [])
|
||||
self.node = node
|
||||
self.finished_deferred = defer.Deferred()
|
||||
# all distance operations in this class only care about the distance
|
||||
# to self.key, so this makes it easier to calculate those
|
||||
self.distance = Distance(key)
|
||||
# The closest known and active node yet found
|
||||
self.closest_node = None if not shortlist else shortlist[0]
|
||||
self.prev_closest_node = None
|
||||
# Shortlist of contact objects (the k closest known contacts to the key from the routing table)
|
||||
self.shortlist = shortlist
|
||||
# The search key
|
||||
self.key = key
|
||||
# The rpc method name (findValue or findNode)
|
||||
self.rpc = rpc
|
||||
# List of active queries; len() indicates number of active probes
|
||||
self.active_probes = []
|
||||
# List of contact (address, port) tuples that have already been queried, includes contacts that didn't reply
|
||||
self.already_contacted = []
|
||||
# A list of found and known-to-be-active remote nodes (Contact objects)
|
||||
self.active_contacts = []
|
||||
# Ensure only one searchIteration call is running at a time
|
||||
self._search_iteration_semaphore = defer.DeferredSemaphore(1)
|
||||
self._iteration_count = 0
|
||||
self.find_value_result = {}
|
||||
self.pending_iteration_calls = []
|
||||
|
||||
@property
|
||||
def is_find_node_request(self):
|
||||
return self.rpc == "findNode"
|
||||
|
||||
@property
|
||||
def is_find_value_request(self):
|
||||
return self.rpc == "findValue"
|
||||
|
||||
def is_closer(self, contact):
|
||||
if not self.closest_node:
|
||||
return True
|
||||
return self.distance.is_closer(contact.id, self.closest_node.id)
|
||||
|
||||
def getContactTriples(self, result):
|
||||
if self.is_find_value_request:
|
||||
contact_triples = result[b'contacts']
|
||||
else:
|
||||
contact_triples = result
|
||||
for contact_tup in contact_triples:
|
||||
if not isinstance(contact_tup, (list, tuple)) or len(contact_tup) != 3:
|
||||
raise ValueError("invalid contact triple")
|
||||
contact_tup[1] = contact_tup[1].decode() # ips are strings
|
||||
return contact_triples
|
||||
|
||||
def sortByDistance(self, contact_list):
|
||||
"""Sort the list of contacts in order by distance from key"""
|
||||
contact_list.sort(key=lambda c: self.distance(c.id))
|
||||
|
||||
def extendShortlist(self, contact, result):
|
||||
# The "raw response" tuple contains the response message and the originating address info
|
||||
originAddress = (contact.address, contact.port)
|
||||
if self.finished_deferred.called:
|
||||
return contact.id
|
||||
if self.node.contact_manager.is_ignored(originAddress):
|
||||
raise ValueError("contact is ignored")
|
||||
if contact.id == self.node.node_id:
|
||||
return contact.id
|
||||
|
||||
if contact not in self.active_contacts:
|
||||
self.active_contacts.append(contact)
|
||||
if contact not in self.shortlist:
|
||||
self.shortlist.append(contact)
|
||||
|
||||
# Now grow extend the (unverified) shortlist with the returned contacts
|
||||
# TODO: some validation on the result (for guarding against attacks)
|
||||
# If we are looking for a value, first see if this result is the value
|
||||
# we are looking for before treating it as a list of contact triples
|
||||
if self.is_find_value_request and self.key in result:
|
||||
# We have found the value
|
||||
for peer in result[self.key]:
|
||||
node_id, host, port = expand_peer(peer)
|
||||
if (host, port) not in self.exclude:
|
||||
self.find_value_result.setdefault(self.key, []).append((node_id, host, port))
|
||||
if self.find_value_result:
|
||||
self.finished_deferred.callback(self.find_value_result)
|
||||
else:
|
||||
if self.is_find_value_request:
|
||||
# We are looking for a value, and the remote node didn't have it
|
||||
# - mark it as the closest "empty" node, if it is
|
||||
# TODO: store to this peer after finding the value as per the kademlia spec
|
||||
if b'closestNodeNoValue' in self.find_value_result:
|
||||
if self.is_closer(contact):
|
||||
self.find_value_result[b'closestNodeNoValue'] = contact
|
||||
else:
|
||||
self.find_value_result[b'closestNodeNoValue'] = contact
|
||||
contactTriples = self.getContactTriples(result)
|
||||
for contactTriple in contactTriples:
|
||||
if (contactTriple[1], contactTriple[2]) in ((c.address, c.port) for c in self.already_contacted):
|
||||
continue
|
||||
elif self.node.contact_manager.is_ignored((contactTriple[1], contactTriple[2])):
|
||||
continue
|
||||
else:
|
||||
found_contact = self.node.contact_manager.make_contact(contactTriple[0], contactTriple[1],
|
||||
contactTriple[2], self.node._protocol)
|
||||
if found_contact not in self.shortlist:
|
||||
self.shortlist.append(found_contact)
|
||||
|
||||
if not self.finished_deferred.called and self.should_stop():
|
||||
self.sortByDistance(self.active_contacts)
|
||||
self.finished_deferred.callback(self.active_contacts[:min(constants.k, len(self.active_contacts))])
|
||||
|
||||
return contact.id
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def probeContact(self, contact):
|
||||
fn = getattr(contact, self.rpc)
|
||||
try:
|
||||
response = yield fn(self.key)
|
||||
result = self.extendShortlist(contact, response)
|
||||
defer.returnValue(result)
|
||||
except (TimeoutError, defer.CancelledError, ValueError, IndexError):
|
||||
defer.returnValue(contact.id)
|
||||
|
||||
def should_stop(self):
|
||||
if self.is_find_value_request:
|
||||
# search stops when it finds a value, let it run
|
||||
return False
|
||||
if self.prev_closest_node and self.closest_node and self.distance.is_closer(self.prev_closest_node.id,
|
||||
self.closest_node.id):
|
||||
# we're getting further away
|
||||
return True
|
||||
if len(self.active_contacts) >= constants.k:
|
||||
# we have enough results
|
||||
return True
|
||||
return False
|
||||
|
||||
# Send parallel, asynchronous FIND_NODE RPCs to the shortlist of contacts
|
||||
def _searchIteration(self):
|
||||
# Sort the discovered active nodes from closest to furthest
|
||||
if len(self.active_contacts):
|
||||
self.sortByDistance(self.active_contacts)
|
||||
self.prev_closest_node = self.closest_node
|
||||
self.closest_node = self.active_contacts[0]
|
||||
|
||||
# Sort the current shortList before contacting other nodes
|
||||
self.sortByDistance(self.shortlist)
|
||||
probes = []
|
||||
already_contacted_addresses = {(c.address, c.port) for c in self.already_contacted}
|
||||
to_remove = []
|
||||
for contact in self.shortlist:
|
||||
if self.node.contact_manager.is_ignored((contact.address, contact.port)):
|
||||
to_remove.append(contact) # a contact became bad during iteration
|
||||
continue
|
||||
if (contact.address, contact.port) not in already_contacted_addresses:
|
||||
self.already_contacted.append(contact)
|
||||
to_remove.append(contact)
|
||||
probe = self.probeContact(contact)
|
||||
probes.append(probe)
|
||||
self.active_probes.append(probe)
|
||||
if len(probes) == constants.alpha:
|
||||
break
|
||||
for contact in to_remove: # these contacts will be re-added to the shortlist when they reply successfully
|
||||
self.shortlist.remove(contact)
|
||||
|
||||
# run the probes
|
||||
if probes:
|
||||
# Schedule the next iteration if there are any active
|
||||
# calls (Kademlia uses loose parallelism)
|
||||
self.searchIteration()
|
||||
|
||||
d = defer.DeferredList(probes, consumeErrors=True)
|
||||
|
||||
def _remove_probes(results):
|
||||
for probe in probes:
|
||||
self.active_probes.remove(probe)
|
||||
return results
|
||||
|
||||
d.addCallback(_remove_probes)
|
||||
|
||||
elif not self.finished_deferred.called and not self.active_probes or self.should_stop():
|
||||
# If no probes were sent, there will not be any improvement, so we're done
|
||||
if self.is_find_value_request:
|
||||
self.finished_deferred.callback(self.find_value_result)
|
||||
else:
|
||||
self.sortByDistance(self.active_contacts)
|
||||
self.finished_deferred.callback(self.active_contacts[:min(constants.k, len(self.active_contacts))])
|
||||
elif not self.finished_deferred.called:
|
||||
# Force the next iteration
|
||||
self.searchIteration()
|
||||
|
||||
def searchIteration(self, delay=constants.iterativeLookupDelay):
|
||||
def _cancel_pending_iterations(result):
|
||||
while self.pending_iteration_calls:
|
||||
canceller = self.pending_iteration_calls.pop()
|
||||
canceller()
|
||||
return result
|
||||
self.finished_deferred.addBoth(_cancel_pending_iterations)
|
||||
self._iteration_count += 1
|
||||
call, cancel = self.node.reactor_callLater(delay, self._search_iteration_semaphore.run, self._searchIteration)
|
||||
self.pending_iteration_calls.append(cancel)
|
||||
|
||||
|
||||
def iterativeFind(node, shortlist, key, rpc, exclude=None):
|
||||
helper = _IterativeFind(node, shortlist, key, rpc, exclude)
|
||||
helper.searchIteration(0)
|
||||
return helper.finished_deferred
|
||||
@@ -1,147 +0,0 @@
|
||||
import logging
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.distance import Distance
|
||||
from lbrynet.dht.error import BucketFull
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KBucket:
|
||||
""" Description - later
|
||||
"""
|
||||
|
||||
def __init__(self, rangeMin, rangeMax, node_id):
|
||||
"""
|
||||
@param rangeMin: The lower boundary for the range in the n-bit ID
|
||||
space covered by this k-bucket
|
||||
@param rangeMax: The upper boundary for the range in the ID space
|
||||
covered by this k-bucket
|
||||
"""
|
||||
self.lastAccessed = 0
|
||||
self.rangeMin = rangeMin
|
||||
self.rangeMax = rangeMax
|
||||
self._contacts = list()
|
||||
self._node_id = node_id
|
||||
|
||||
def addContact(self, contact):
|
||||
""" Add contact to _contact list in the right order. This will move the
|
||||
contact to the end of the k-bucket if it is already present.
|
||||
|
||||
@raise kademlia.kbucket.BucketFull: Raised when the bucket is full and
|
||||
the contact isn't in the bucket
|
||||
already
|
||||
|
||||
@param contact: The contact to add
|
||||
@type contact: dht.contact._Contact
|
||||
"""
|
||||
if contact in self._contacts:
|
||||
# Move the existing contact to the end of the list
|
||||
# - using the new contact to allow add-on data
|
||||
# (e.g. optimization-specific stuff) to pe updated as well
|
||||
self._contacts.remove(contact)
|
||||
self._contacts.append(contact)
|
||||
elif len(self._contacts) < constants.k:
|
||||
self._contacts.append(contact)
|
||||
else:
|
||||
raise BucketFull("No space in bucket to insert contact")
|
||||
|
||||
def getContact(self, contactID):
|
||||
"""Get the contact specified node ID
|
||||
|
||||
@raise IndexError: raised if the contact is not in the bucket
|
||||
|
||||
@param contactID: the node id of the contact to retrieve
|
||||
@type contactID: str
|
||||
|
||||
@rtype: dht.contact._Contact
|
||||
"""
|
||||
for contact in self._contacts:
|
||||
if contact.id == contactID:
|
||||
return contact
|
||||
raise IndexError(contactID)
|
||||
|
||||
def getContacts(self, count=-1, excludeContact=None, sort_distance_to=None):
|
||||
""" Returns a list containing up to the first count number of contacts
|
||||
|
||||
@param count: The amount of contacts to return (if 0 or less, return
|
||||
all contacts)
|
||||
@type count: int
|
||||
@param excludeContact: A node id to exclude; if this contact is in
|
||||
the list of returned values, it will be
|
||||
discarded before returning. If a C{str} is
|
||||
passed as this argument, it must be the
|
||||
contact's ID.
|
||||
@type excludeContact: str
|
||||
|
||||
@param sort_distance_to: Sort distance to the id, defaulting to the parent node id. If False don't
|
||||
sort the contacts
|
||||
|
||||
@raise IndexError: If the number of requested contacts is too large
|
||||
|
||||
@return: Return up to the first count number of contacts in a list
|
||||
If no contacts are present an empty is returned
|
||||
@rtype: list
|
||||
"""
|
||||
contacts = [contact for contact in self._contacts if contact.id != excludeContact]
|
||||
|
||||
# Return all contacts in bucket
|
||||
if count <= 0:
|
||||
count = len(contacts)
|
||||
|
||||
# Get current contact number
|
||||
currentLen = len(contacts)
|
||||
|
||||
# If count greater than k - return only k contacts
|
||||
if count > constants.k:
|
||||
count = constants.k
|
||||
|
||||
if not currentLen:
|
||||
return contacts
|
||||
|
||||
if sort_distance_to is False:
|
||||
pass
|
||||
else:
|
||||
sort_distance_to = sort_distance_to or self._node_id
|
||||
contacts.sort(key=lambda c: Distance(sort_distance_to)(c.id))
|
||||
|
||||
return contacts[:min(currentLen, count)]
|
||||
|
||||
def getBadOrUnknownContacts(self):
|
||||
contacts = self.getContacts(sort_distance_to=False)
|
||||
results = [contact for contact in contacts if contact.contact_is_good is False]
|
||||
results.extend(contact for contact in contacts if contact.contact_is_good is None)
|
||||
return results
|
||||
|
||||
def removeContact(self, contact):
|
||||
""" Remove the contact from the bucket
|
||||
|
||||
@param contact: The contact to remove
|
||||
@type contact: dht.contact._Contact
|
||||
|
||||
@raise ValueError: The specified contact is not in this bucket
|
||||
"""
|
||||
self._contacts.remove(contact)
|
||||
|
||||
def keyInRange(self, key):
|
||||
""" Tests whether the specified key (i.e. node ID) is in the range
|
||||
of the n-bit ID space covered by this k-bucket (in otherwords, it
|
||||
returns whether or not the specified key should be placed in this
|
||||
k-bucket)
|
||||
|
||||
@param key: The key to test
|
||||
@type key: str or int
|
||||
|
||||
@return: C{True} if the key is in this k-bucket's range, or C{False}
|
||||
if not.
|
||||
@rtype: bool
|
||||
"""
|
||||
if isinstance(key, bytes):
|
||||
key = int.from_bytes(key, 'big')
|
||||
return self.rangeMin <= key < self.rangeMax
|
||||
|
||||
def __len__(self):
|
||||
return len(self._contacts)
|
||||
|
||||
def __contains__(self, item):
|
||||
return item in self._contacts
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# This library is free software, distributed under the terms of
|
||||
# the GNU Lesser General Public License Version 3, or any later version.
|
||||
# See the COPYING file included in this archive
|
||||
#
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
from lbrynet.dht import msgtypes
|
||||
|
||||
|
||||
class MessageTranslator:
|
||||
""" Interface for RPC message translators/formatters
|
||||
|
||||
Classes inheriting from this should provide a translation services between
|
||||
the classes used internally by this Kademlia implementation and the actual
|
||||
data that is transmitted between nodes.
|
||||
"""
|
||||
|
||||
def fromPrimitive(self, msgPrimitive):
|
||||
""" Create an RPC Message from a message's string representation
|
||||
|
||||
@param msgPrimitive: The unencoded primitive representation of a message
|
||||
@type msgPrimitive: str, int, list or dict
|
||||
|
||||
@return: The translated message object
|
||||
@rtype: entangled.kademlia.msgtypes.Message
|
||||
"""
|
||||
|
||||
def toPrimitive(self, message):
|
||||
""" Create a string representation of a message
|
||||
|
||||
@param message: The message object
|
||||
@type message: msgtypes.Message
|
||||
|
||||
@return: The message's primitive representation in a particular
|
||||
messaging format
|
||||
@rtype: str, int, list or dict
|
||||
"""
|
||||
|
||||
|
||||
class DefaultFormat(MessageTranslator):
|
||||
""" The default on-the-wire message format for this library """
|
||||
typeRequest, typeResponse, typeError = range(3)
|
||||
headerType, headerMsgID, headerNodeID, headerPayload, headerArgs = range(5) # TODO: make str
|
||||
|
||||
@staticmethod
|
||||
def get(primitive, key):
|
||||
try:
|
||||
return primitive[key]
|
||||
except KeyError:
|
||||
return primitive[str(key)] # TODO: switch to int()
|
||||
|
||||
def fromPrimitive(self, msgPrimitive):
|
||||
msgType = self.get(msgPrimitive, self.headerType)
|
||||
if msgType == self.typeRequest:
|
||||
msg = msgtypes.RequestMessage(self.get(msgPrimitive, self.headerNodeID),
|
||||
self.get(msgPrimitive, self.headerPayload),
|
||||
self.get(msgPrimitive, self.headerArgs),
|
||||
self.get(msgPrimitive, self.headerMsgID))
|
||||
elif msgType == self.typeResponse:
|
||||
msg = msgtypes.ResponseMessage(self.get(msgPrimitive, self.headerMsgID),
|
||||
self.get(msgPrimitive, self.headerNodeID),
|
||||
self.get(msgPrimitive, self.headerPayload))
|
||||
elif msgType == self.typeError:
|
||||
msg = msgtypes.ErrorMessage(self.get(msgPrimitive, self.headerMsgID),
|
||||
self.get(msgPrimitive, self.headerNodeID),
|
||||
self.get(msgPrimitive, self.headerPayload),
|
||||
self.get(msgPrimitive, self.headerArgs))
|
||||
else:
|
||||
# Unknown message, no payload
|
||||
msg = msgtypes.Message(msgPrimitive[self.headerMsgID], msgPrimitive[self.headerNodeID])
|
||||
return msg
|
||||
|
||||
def toPrimitive(self, message):
|
||||
msg = {self.headerMsgID: message.id,
|
||||
self.headerNodeID: message.nodeID}
|
||||
if isinstance(message, msgtypes.RequestMessage):
|
||||
msg[self.headerType] = self.typeRequest
|
||||
msg[self.headerPayload] = message.request
|
||||
msg[self.headerArgs] = message.args
|
||||
elif isinstance(message, msgtypes.ErrorMessage):
|
||||
msg[self.headerType] = self.typeError
|
||||
msg[self.headerPayload] = message.exceptionType
|
||||
msg[self.headerArgs] = message.response
|
||||
elif isinstance(message, msgtypes.ResponseMessage):
|
||||
msg[self.headerType] = self.typeResponse
|
||||
msg[self.headerPayload] = message.response
|
||||
return msg
|
||||
@@ -1,52 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# This library is free software, distributed under the terms of
|
||||
# the GNU Lesser General Public License Version 3, or any later version.
|
||||
# See the COPYING file included in this archive
|
||||
#
|
||||
# The docstrings in this module contain epytext markup; API documentation
|
||||
# may be created by processing this file with epydoc: http://epydoc.sf.net
|
||||
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.dht import constants
|
||||
|
||||
|
||||
class Message:
|
||||
""" Base class for messages - all "unknown" messages use this class """
|
||||
|
||||
def __init__(self, rpcID, nodeID):
|
||||
if len(rpcID) != constants.rpc_id_length:
|
||||
raise ValueError("invalid rpc id: %i bytes (expected 20)" % len(rpcID))
|
||||
if len(nodeID) != constants.key_bits // 8:
|
||||
raise ValueError("invalid node id: %i bytes (expected 48)" % len(nodeID))
|
||||
self.id = rpcID
|
||||
self.nodeID = nodeID
|
||||
|
||||
|
||||
class RequestMessage(Message):
|
||||
""" Message containing an RPC request """
|
||||
|
||||
def __init__(self, nodeID, method, methodArgs, rpcID=None):
|
||||
if rpcID is None:
|
||||
rpcID = generate_id()[:constants.rpc_id_length]
|
||||
super().__init__(rpcID, nodeID)
|
||||
self.request = method
|
||||
self.args = methodArgs
|
||||
|
||||
|
||||
class ResponseMessage(Message):
|
||||
""" Message containing the result from a successful RPC request """
|
||||
|
||||
def __init__(self, rpcID, nodeID, response):
|
||||
super().__init__(rpcID, nodeID)
|
||||
self.response = response
|
||||
|
||||
|
||||
class ErrorMessage(ResponseMessage):
|
||||
""" Message containing the error from an unsuccessful RPC request """
|
||||
|
||||
def __init__(self, rpcID, nodeID, exceptionType, errorMessage):
|
||||
super().__init__(rpcID, nodeID, errorMessage)
|
||||
if isinstance(exceptionType, type):
|
||||
exceptionType = (f'{exceptionType.__module__}.{exceptionType.__name__}').encode()
|
||||
self.exceptionType = exceptionType
|
||||
@@ -1,649 +1,226 @@
|
||||
import binascii
|
||||
import hashlib
|
||||
import logging
|
||||
from functools import reduce
|
||||
import asyncio
|
||||
import typing
|
||||
import socket
|
||||
import binascii
|
||||
import contextlib
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.async_generator_junction import AsyncGeneratorJunction
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
from lbrynet.dht.protocol.iterative_find import IterativeNodeFinder, IterativeValueFinder
|
||||
from lbrynet.dht.protocol.protocol import KademliaProtocol
|
||||
from lbrynet.dht.peer import KademliaPeer
|
||||
|
||||
from twisted.internet import defer, error, task
|
||||
|
||||
from lbrynet.utils import generate_id, DeferredDict
|
||||
from lbrynet.dht.call_later_manager import CallLaterManager
|
||||
from lbrynet.dht.error import TimeoutError
|
||||
from lbrynet.dht import constants, routingtable, datastore, protocol
|
||||
from lbrynet.dht.contact import ContactManager
|
||||
from lbrynet.dht.iterativefind import iterativeFind
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.peer import PeerManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def rpcmethod(func):
|
||||
""" Decorator to expose Node methods as remote procedure calls
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port)
|
||||
self.listening_port: asyncio.DatagramTransport = None
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
self._refresh_task: asyncio.Task = None
|
||||
|
||||
Apply this decorator to methods in the Node class (or a subclass) in order
|
||||
to make them remotely callable via the DHT's RPC mechanism.
|
||||
"""
|
||||
func.rpcmethod = True
|
||||
return func
|
||||
async def refresh_node(self):
|
||||
while True:
|
||||
# remove peers with expired blob announcements from the datastore
|
||||
self.protocol.data_store.removed_expired_peers()
|
||||
|
||||
total_peers: typing.List['KademliaPeer'] = []
|
||||
# add all peers in the routing table
|
||||
total_peers.extend(self.protocol.routing_table.get_peers())
|
||||
# add all the peers who have announed blobs to us
|
||||
total_peers.extend(self.protocol.data_store.get_storing_contacts())
|
||||
|
||||
class MockKademliaHelper:
|
||||
def __init__(self, clock=None, callLater=None, resolve=None, listenUDP=None):
|
||||
if not listenUDP or not resolve or not callLater or not clock:
|
||||
from twisted.internet import reactor
|
||||
listenUDP = listenUDP or reactor.listenUDP
|
||||
resolve = resolve or reactor.resolve
|
||||
callLater = callLater or reactor.callLater
|
||||
clock = clock or reactor
|
||||
# get ids falling in the midpoint of each bucket that hasn't been recently updated
|
||||
node_ids = self.protocol.routing_table.get_refresh_list(0, True)
|
||||
# if we have 3 or fewer populated buckets get two random ids in the range of each to try and
|
||||
# populate/split the buckets further
|
||||
buckets_with_contacts = self.protocol.routing_table.buckets_with_contacts()
|
||||
if buckets_with_contacts <= 3:
|
||||
for i in range(buckets_with_contacts):
|
||||
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
|
||||
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
|
||||
|
||||
self.clock = clock
|
||||
self.contact_manager = ContactManager(self.clock.seconds)
|
||||
self.reactor_listenUDP = listenUDP
|
||||
self.reactor_resolve = resolve
|
||||
self.call_later_manager = CallLaterManager(callLater)
|
||||
self.reactor_callLater = self.call_later_manager.call_later
|
||||
self.reactor_callSoon = self.call_later_manager.call_soon
|
||||
|
||||
self._listeningPort = None # object implementing Twisted
|
||||
# IListeningPort This will contain a deferred created when
|
||||
# joining the network, to enable publishing/retrieving
|
||||
# information from the DHT as soon as the node is part of the
|
||||
# network (add callbacks to this deferred if scheduling such
|
||||
# operations before the node has finished joining the network)
|
||||
|
||||
def get_looping_call(self, fn, *args, **kwargs):
|
||||
lc = task.LoopingCall(fn, *args, **kwargs)
|
||||
lc.clock = self.clock
|
||||
return lc
|
||||
|
||||
def safe_stop_looping_call(self, lc):
|
||||
if lc and lc.running:
|
||||
return lc.stop()
|
||||
return defer.succeed(None)
|
||||
|
||||
def safe_start_looping_call(self, lc, t):
|
||||
if lc and not lc.running:
|
||||
lc.start(t)
|
||||
|
||||
|
||||
class Node(MockKademliaHelper):
|
||||
""" Local node in the Kademlia network
|
||||
|
||||
This class represents a single local node in a Kademlia network; in other
|
||||
words, this class encapsulates an Entangled-using application's "presence"
|
||||
in a Kademlia network.
|
||||
|
||||
In Entangled, all interactions with the Kademlia network by a client
|
||||
application is performed via this class (or a subclass).
|
||||
"""
|
||||
|
||||
def __init__(self, node_id=None, udpPort=4000, dataStore=None,
|
||||
routingTableClass=None, networkProtocol=None,
|
||||
externalIP=None, peerPort=3333, listenUDP=None,
|
||||
callLater=None, resolve=None, clock=None,
|
||||
interface='', externalUDPPort=None):
|
||||
"""
|
||||
@param dataStore: The data store to use. This must be class inheriting
|
||||
from the C{DataStore} interface (or providing the
|
||||
same API). How the data store manages its data
|
||||
internally is up to the implementation of that data
|
||||
store.
|
||||
@type dataStore: entangled.kademlia.datastore.DataStore
|
||||
@param routingTable: The routing table class to use. Since there exists
|
||||
some ambiguity as to how the routing table should be
|
||||
implemented in Kademlia, a different routing table
|
||||
may be used, as long as the appropriate API is
|
||||
exposed. This should be a class, not an object,
|
||||
in order to allow the Node to pass an
|
||||
auto-generated node ID to the routingtable object
|
||||
upon instantiation (if necessary).
|
||||
@type routingTable: entangled.kademlia.routingtable.RoutingTable
|
||||
@param networkProtocol: The network protocol to use. This can be
|
||||
overridden from the default to (for example)
|
||||
change the format of the physical RPC messages
|
||||
being transmitted.
|
||||
@type networkProtocol: entangled.kademlia.protocol.KademliaProtocol
|
||||
@param externalIP: the IP at which this node can be contacted
|
||||
@param peerPort: the port at which this node announces it has a blob for
|
||||
"""
|
||||
|
||||
super().__init__(clock, callLater, resolve, listenUDP)
|
||||
self.node_id = node_id or self._generateID()
|
||||
self.port = udpPort
|
||||
self._listen_interface = interface
|
||||
self._change_token_lc = self.get_looping_call(self.change_token)
|
||||
self._refresh_node_lc = self.get_looping_call(self._refreshNode)
|
||||
self._refresh_contacts_lc = self.get_looping_call(self._refreshContacts)
|
||||
|
||||
# Create k-buckets (for storing contacts)
|
||||
if routingTableClass is None:
|
||||
self._routingTable = routingtable.TreeRoutingTable(self.node_id, self.clock.seconds)
|
||||
else:
|
||||
self._routingTable = routingTableClass(self.node_id, self.clock.seconds)
|
||||
|
||||
self._protocol = networkProtocol or protocol.KademliaProtocol(self)
|
||||
self.token_secret = self._generateID()
|
||||
self.old_token_secret = None
|
||||
self.externalIP = externalIP
|
||||
self.peerPort = peerPort
|
||||
self.externalUDPPort = externalUDPPort or self.port
|
||||
self._dataStore = dataStore or datastore.DictDataStore(self.clock.seconds)
|
||||
self._join_deferred = None
|
||||
|
||||
#def __del__(self):
|
||||
# log.warning("unclean shutdown of the dht node")
|
||||
# if hasattr(self, "_listeningPort") and self._listeningPort is not None:
|
||||
# self._listeningPort.stopListening()
|
||||
|
||||
def __str__(self):
|
||||
return '<%s.%s object; ID: %s, IP address: %s, UDP port: %d>' % (
|
||||
self.__module__, self.__class__.__name__, binascii.hexlify(self.node_id), self.externalIP, self.port)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
# stop LoopingCalls:
|
||||
yield self.safe_stop_looping_call(self._refresh_node_lc)
|
||||
yield self.safe_stop_looping_call(self._change_token_lc)
|
||||
yield self.safe_stop_looping_call(self._refresh_contacts_lc)
|
||||
if self._listeningPort is not None:
|
||||
yield self._listeningPort.stopListening()
|
||||
self._listeningPort = None
|
||||
|
||||
def start_listening(self):
|
||||
if not self._listeningPort:
|
||||
try:
|
||||
self._listeningPort = self.reactor_listenUDP(self.port, self._protocol,
|
||||
interface=self._listen_interface)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running." % str(e))
|
||||
else:
|
||||
log.warning("Already bound to port %s", self._listeningPort)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def joinNetwork(self, known_node_addresses=(('jack.lbry.tech', 4455), )):
|
||||
"""
|
||||
Attempt to join the dht, retry every 30 seconds if unsuccessful
|
||||
:param known_node_addresses: [(str, int)] list of hostnames and ports for known dht seed nodes
|
||||
"""
|
||||
|
||||
self._join_deferred = defer.Deferred()
|
||||
known_node_resolution = {}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _resolve_seeds():
|
||||
result = {}
|
||||
for host, port in known_node_addresses:
|
||||
node_address = yield self.reactor_resolve(host)
|
||||
result[(host, port)] = node_address
|
||||
defer.returnValue(result)
|
||||
|
||||
if not known_node_resolution:
|
||||
known_node_resolution = yield _resolve_seeds()
|
||||
# we are one of the seed nodes, don't add ourselves
|
||||
if (self.externalIP, self.port) in known_node_resolution.values():
|
||||
del known_node_resolution[(self.externalIP, self.port)]
|
||||
known_node_addresses.remove((self.externalIP, self.port))
|
||||
|
||||
def _ping_contacts(contacts):
|
||||
d = DeferredDict({contact: contact.ping() for contact in contacts}, consumeErrors=True)
|
||||
d.addErrback(lambda err: err.trap(TimeoutError))
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _initialize_routing():
|
||||
bootstrap_contacts = []
|
||||
contact_addresses = {(c.address, c.port): c for c in self.contacts}
|
||||
for (host, port), ip_address in known_node_resolution.items():
|
||||
if (host, port) not in contact_addresses:
|
||||
# Create temporary contact information for the list of addresses of known nodes
|
||||
# The contact node id will be set with the responding node id when we initialize it to None
|
||||
contact = self.contact_manager.make_contact(None, ip_address, port, self._protocol)
|
||||
bootstrap_contacts.append(contact)
|
||||
else:
|
||||
for contact in self.contacts:
|
||||
if contact.address == ip_address and contact.port == port:
|
||||
if not contact.id:
|
||||
bootstrap_contacts.append(contact)
|
||||
break
|
||||
if not bootstrap_contacts:
|
||||
log.warning("no bootstrap contacts to ping")
|
||||
ping_result = yield _ping_contacts(bootstrap_contacts)
|
||||
shortlist = list(ping_result.keys())
|
||||
if not shortlist:
|
||||
log.warning("failed to ping %i bootstrap contacts", len(bootstrap_contacts))
|
||||
defer.returnValue(None)
|
||||
if self.protocol.routing_table.get_peers():
|
||||
# if we have node ids to look up, perform the iterative search until we have k results
|
||||
while node_ids:
|
||||
peers = await self.peer_search(node_ids.pop())
|
||||
total_peers.extend(peers)
|
||||
else:
|
||||
# find the closest peers to us
|
||||
closest = yield self._iterativeFind(self.node_id, shortlist if not self.contacts else None)
|
||||
yield _ping_contacts(closest)
|
||||
# query random hashes in our bucket key ranges to fill or split them
|
||||
random_ids_in_range = self._routingTable.getRefreshList()
|
||||
while random_ids_in_range:
|
||||
yield self.iterativeFindNode(random_ids_in_range.pop())
|
||||
defer.returnValue(None)
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.refresh_interval // 4, fut.set_result, None)
|
||||
await fut
|
||||
continue
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _iterative_join(joined_d=None, last_buckets_with_contacts=None):
|
||||
log.info("Attempting to join the DHT network, %i contacts known so far", len(self.contacts))
|
||||
joined_d = joined_d or defer.Deferred()
|
||||
yield _initialize_routing()
|
||||
buckets_with_contacts = self.bucketsWithContacts()
|
||||
if last_buckets_with_contacts and last_buckets_with_contacts == buckets_with_contacts:
|
||||
if not joined_d.called:
|
||||
joined_d.callback(True)
|
||||
elif buckets_with_contacts < 4:
|
||||
self.reactor_callLater(0, _iterative_join, joined_d, buckets_with_contacts)
|
||||
elif not joined_d.called:
|
||||
joined_d.callback(None)
|
||||
yield joined_d
|
||||
if not self._join_deferred.called:
|
||||
self._join_deferred.callback(True)
|
||||
defer.returnValue(None)
|
||||
# ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated
|
||||
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
|
||||
if to_ping:
|
||||
await self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
|
||||
yield _iterative_join()
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
|
||||
await fut
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self, known_node_addresses=None, block_on_join=False):
|
||||
""" Causes the Node to attempt to join the DHT network by contacting the
|
||||
known DHT nodes. This can be called multiple times if the previous attempt
|
||||
has failed or if the Node has lost all the contacts.
|
||||
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
|
||||
announced_to_node_ids = []
|
||||
while not announced_to_node_ids:
|
||||
hash_value = binascii.unhexlify(blob_hash.encode())
|
||||
assert len(hash_value) == constants.hash_length
|
||||
peers = await self.peer_search(hash_value)
|
||||
|
||||
@param known_node_addresses: A sequence of tuples containing IP address
|
||||
information for existing nodes on the
|
||||
Kademlia network, in the format:
|
||||
C{(<ip address>, (udp port>)}
|
||||
@type known_node_addresses: list
|
||||
"""
|
||||
if not self.protocol.external_ip:
|
||||
raise Exception("Cannot determine external IP")
|
||||
log.info("Store to %i peers", len(peers))
|
||||
log.info(peers)
|
||||
for peer in peers:
|
||||
log.info("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
stored_to_tup = await asyncio.gather(
|
||||
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
|
||||
)
|
||||
announced_to_node_ids.extend([node_id for node_id, contacted in stored_to_tup if contacted])
|
||||
log.info("Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
|
||||
len(announced_to_node_ids), len(peers))
|
||||
return announced_to_node_ids
|
||||
|
||||
self.start_listening()
|
||||
yield self._protocol._listening
|
||||
# TODO: Refresh all k-buckets further away than this node's closest neighbour
|
||||
d = self.joinNetwork(known_node_addresses or [])
|
||||
d.addCallback(lambda _: self.start_looping_calls())
|
||||
d.addCallback(lambda _: log.info("Joined the dht"))
|
||||
if block_on_join:
|
||||
yield d
|
||||
def stop(self) -> None:
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
if self._join_task:
|
||||
self._join_task.cancel()
|
||||
if self._refresh_task and not (self._refresh_task.done() or self._refresh_task.cancelled()):
|
||||
self._refresh_task.cancel()
|
||||
if self.protocol and self.protocol.ping_queue.running:
|
||||
self.protocol.ping_queue.stop()
|
||||
if self.listening_port is not None:
|
||||
self.listening_port.close()
|
||||
self._join_task = None
|
||||
self.listening_port = None
|
||||
log.info("Stopped DHT node")
|
||||
|
||||
def start_looping_calls(self):
|
||||
self.safe_start_looping_call(self._change_token_lc, constants.tokenSecretChangeInterval)
|
||||
# Start refreshing k-buckets periodically, if necessary
|
||||
self.safe_start_looping_call(self._refresh_node_lc, constants.checkRefreshInterval)
|
||||
self.safe_start_looping_call(self._refresh_contacts_lc, 60)
|
||||
async def start_listening(self, interface: str = '') -> None:
|
||||
if not self.listening_port:
|
||||
self.listening_port, _ = await self.loop.create_datagram_endpoint(
|
||||
lambda: self.protocol, (interface, self.internal_udp_port)
|
||||
)
|
||||
log.info("DHT node listening on UDP %s:%i", interface, self.internal_udp_port)
|
||||
else:
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
@property
|
||||
def contacts(self):
|
||||
def _inner():
|
||||
for i in range(len(self._routingTable._buckets)):
|
||||
for contact in self._routingTable._buckets[i]._contacts:
|
||||
yield contact
|
||||
return list(_inner())
|
||||
async def join_network(self, interface: typing.Optional[str] = '',
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
known_node_addresses: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
if not self.listening_port:
|
||||
await self.start_listening(interface)
|
||||
self.protocol.ping_queue.start()
|
||||
self._refresh_task = self.loop.create_task(self.refresh_node())
|
||||
|
||||
def hasContacts(self):
|
||||
for bucket in self._routingTable._buckets:
|
||||
if bucket._contacts:
|
||||
return True
|
||||
return False
|
||||
known_node_addresses = known_node_addresses or []
|
||||
if known_node_urls:
|
||||
for host, port in known_node_urls:
|
||||
info = await self.loop.getaddrinfo(
|
||||
host, 'https',
|
||||
proto=socket.IPPROTO_TCP,
|
||||
)
|
||||
if (info[0][4][0], port) not in known_node_addresses:
|
||||
known_node_addresses.append((info[0][4][0], port))
|
||||
futs = []
|
||||
for address, port in known_node_addresses:
|
||||
peer = self.protocol.get_rpc_peer(KademliaPeer(self.loop, address, udp_port=port))
|
||||
futs.append(peer.ping())
|
||||
if futs:
|
||||
await asyncio.wait(futs, loop=self.loop)
|
||||
|
||||
def bucketsWithContacts(self):
|
||||
return self._routingTable.bucketsWithContacts()
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=16) as junction:
|
||||
async for peers in junction:
|
||||
for peer in peers:
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
self.joined.set()
|
||||
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def storeToContact(self, blob_hash, contact):
|
||||
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
|
||||
self._join_task = self.loop.create_task(
|
||||
self.join_network(
|
||||
interface=interface, known_node_urls=known_node_urls
|
||||
)
|
||||
)
|
||||
|
||||
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List] = None,
|
||||
bottom_out_limit: int = constants.bottom_out_limit,
|
||||
max_results: int = constants.k) -> IterativeNodeFinder:
|
||||
|
||||
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
|
||||
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List] = None,
|
||||
bottom_out_limit: int = 40,
|
||||
max_results: int = -1) -> IterativeValueFinder:
|
||||
|
||||
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def stream_peer_search_junction(self, hash_queue: asyncio.Queue, bottom_out_limit=20,
|
||||
max_results=-1) -> AsyncGeneratorJunction:
|
||||
peer_generator = AsyncGeneratorJunction(self.loop)
|
||||
|
||||
async def _add_hashes_from_queue():
|
||||
while True:
|
||||
try:
|
||||
blob_hash = await hash_queue.get()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
peer_generator.add_generator(
|
||||
self.get_iterative_value_finder(
|
||||
binascii.unhexlify(blob_hash.encode()), bottom_out_limit=bottom_out_limit,
|
||||
max_results=max_results
|
||||
)
|
||||
)
|
||||
add_hashes_task = self.loop.create_task(_add_hashes_from_queue())
|
||||
try:
|
||||
if not contact.token:
|
||||
yield contact.findValue(blob_hash)
|
||||
res = yield contact.store(blob_hash, contact.token, self.peerPort, self.node_id, 0)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.debug("Stored %s to %s (%s)", binascii.hexlify(blob_hash), contact.log_id(), contact.address)
|
||||
return True
|
||||
except protocol.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s",
|
||||
binascii.hexlify(blob_hash), contact.log_id())
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
contact.update_token(None)
|
||||
log.error("Unexpected error while storing blob_hash %s at %s: %s",
|
||||
binascii.hexlify(blob_hash), contact, err)
|
||||
return False
|
||||
async with peer_generator as junction:
|
||||
yield junction
|
||||
await peer_generator.finished.wait()
|
||||
except asyncio.CancelledError:
|
||||
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
|
||||
add_hashes_task.cancel()
|
||||
raise
|
||||
finally:
|
||||
if add_hashes_task and not (add_hashes_task.done() or add_hashes_task.cancelled()):
|
||||
add_hashes_task.cancel()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def announceHaveBlob(self, blob_hash):
|
||||
contacts = yield self.iterativeFindNode(blob_hash)
|
||||
def peer_search_junction(self, node_id: bytes, max_results=constants.k*2,
|
||||
bottom_out_limit=20) -> AsyncGeneratorJunction:
|
||||
peer_generator = AsyncGeneratorJunction(self.loop)
|
||||
peer_generator.add_generator(
|
||||
self.get_iterative_node_finder(
|
||||
node_id, bottom_out_limit=bottom_out_limit, max_results=max_results
|
||||
)
|
||||
)
|
||||
return peer_generator
|
||||
|
||||
if not self.externalIP:
|
||||
raise Exception("Cannot determine external IP: %s" % self.externalIP)
|
||||
stored_to = yield DeferredDict({contact: self.storeToContact(blob_hash, contact) for contact in contacts})
|
||||
contacted_node_ids = [binascii.hexlify(contact.id) for contact in stored_to.keys() if stored_to[contact]]
|
||||
log.debug("Stored %s to %i of %i attempted peers", binascii.hexlify(blob_hash),
|
||||
len(contacted_node_ids), len(contacts))
|
||||
defer.returnValue(contacted_node_ids)
|
||||
|
||||
def change_token(self):
|
||||
self.old_token_secret = self.token_secret
|
||||
self.token_secret = self._generateID()
|
||||
|
||||
def make_token(self, compact_ip):
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.token_secret + compact_ip)
|
||||
return h.digest()
|
||||
|
||||
def verify_token(self, token, compact_ip):
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.token_secret + compact_ip)
|
||||
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.old_token_secret + compact_ip)
|
||||
if not token == h.digest():
|
||||
return False
|
||||
return True
|
||||
|
||||
def iterativeFindNode(self, key):
|
||||
""" The basic Kademlia node lookup operation
|
||||
|
||||
Call this to find a remote node in the P2P overlay network.
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
|
||||
@return: This immediately returns a deferred object, which will return
|
||||
a list of k "closest" contacts (C{kademlia.contact.Contact}
|
||||
objects) to the specified key as soon as the operation is
|
||||
finished.
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
return self._iterativeFind(key)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def iterativeFindValue(self, key, exclude=None):
|
||||
""" The Kademlia search operation (deterministic)
|
||||
|
||||
Call this to retrieve data from the DHT.
|
||||
|
||||
@param key: the n-bit key (i.e. the value ID) to search for
|
||||
@type key: str
|
||||
|
||||
@return: This immediately returns a deferred object, which will return
|
||||
either one of two things:
|
||||
- If the value was found, it will return a Python
|
||||
dictionary containing the searched-for key (the C{key}
|
||||
parameter passed to this method), and its associated
|
||||
value, in the format:
|
||||
C{<str>key: <str>data_value}
|
||||
- If the value was not found, it will return a list of k
|
||||
"closest" contacts (C{kademlia.contact.Contact} objects)
|
||||
to the specified key
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid key length!")
|
||||
|
||||
# Execute the search
|
||||
find_result = yield self._iterativeFind(key, rpc='findValue', exclude=exclude)
|
||||
if isinstance(find_result, dict):
|
||||
# We have found the value; now see who was the closest contact without it...
|
||||
# ...and store the key/value pair
|
||||
pass
|
||||
else:
|
||||
# The value wasn't found, but a list of contacts was returned
|
||||
# Now, see if we have the value (it might seem wasteful to search on the network
|
||||
# first, but it ensures that all values are properly propagated through the
|
||||
# network
|
||||
if self._dataStore.hasPeersForBlob(key):
|
||||
# Ok, we have the value locally, so use that
|
||||
# Send this value to the closest node without it
|
||||
peers = self._dataStore.getPeersForBlob(key)
|
||||
find_result = {key: peers}
|
||||
else:
|
||||
pass
|
||||
|
||||
defer.returnValue(list(set(find_result.get(key, []) if find_result else [])))
|
||||
# TODO: get this working
|
||||
# if 'closestNodeNoValue' in find_result:
|
||||
# closest_node_without_value = find_result['closestNodeNoValue']
|
||||
# try:
|
||||
# response, address = yield closest_node_without_value.findValue(key, rawResponse=True)
|
||||
# yield closest_node_without_value.store(key, response.response['token'], self.peerPort)
|
||||
# except TimeoutError:
|
||||
# pass
|
||||
|
||||
def addContact(self, contact):
|
||||
""" Add/update the given contact; simple wrapper for the same method
|
||||
in this object's RoutingTable object
|
||||
|
||||
@param contact: The contact to add to this node's k-buckets
|
||||
@type contact: kademlia.contact.Contact
|
||||
"""
|
||||
return self._routingTable.addContact(contact)
|
||||
|
||||
def removeContact(self, contact):
|
||||
""" Remove the contact with the specified node ID from this node's
|
||||
table of known nodes. This is a simple wrapper for the same method
|
||||
in this object's RoutingTable object
|
||||
|
||||
@param contact: The Contact object to remove
|
||||
@type contact: _Contact
|
||||
"""
|
||||
self._routingTable.removeContact(contact)
|
||||
|
||||
def findContact(self, contactID):
|
||||
""" Find a entangled.kademlia.contact.Contact object for the specified
|
||||
cotact ID
|
||||
|
||||
@param contactID: The contact ID of the required Contact object
|
||||
@type contactID: str
|
||||
|
||||
@return: Contact object of remote node with the specified node ID,
|
||||
or None if the contact was not found
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
try:
|
||||
df = defer.succeed(self._routingTable.getContact(contactID))
|
||||
except (ValueError, IndexError):
|
||||
df = self.iterativeFindNode(contactID)
|
||||
df.addCallback(lambda nodes: ([node for node in nodes if node.id == contactID] or (None,))[0])
|
||||
return df
|
||||
|
||||
@rpcmethod
|
||||
def ping(self):
|
||||
""" Used to verify contact between two Kademlia nodes
|
||||
|
||||
@rtype: str
|
||||
"""
|
||||
return b'pong'
|
||||
|
||||
@rpcmethod
|
||||
def store(self, rpc_contact, blob_hash, token, port, originalPublisherID, age):
|
||||
""" Store the received data in this node's local datastore
|
||||
|
||||
@param blob_hash: The hash of the data
|
||||
@type blob_hash: str
|
||||
|
||||
@param token: The token we previously returned when this contact sent us a findValue
|
||||
@type token: str
|
||||
|
||||
@param port: The TCP port the contact is listening on for requests for this blob (the peerPort)
|
||||
@type port: int
|
||||
|
||||
@param originalPublisherID: The node ID of the node that is the publisher of the data
|
||||
@type originalPublisherID: str
|
||||
|
||||
@param age: The relative age of the data (time in seconds since it was
|
||||
originally published). Note that the original publish time
|
||||
isn't actually given, to compensate for clock skew between
|
||||
different nodes.
|
||||
@type age: int
|
||||
|
||||
@rtype: str
|
||||
"""
|
||||
|
||||
if originalPublisherID is None:
|
||||
originalPublisherID = rpc_contact.id
|
||||
compact_ip = rpc_contact.compact_ip()
|
||||
if self.clock.seconds() - self._protocol.started_listening_time < constants.tokenSecretChangeInterval:
|
||||
pass
|
||||
elif not self.verify_token(token, compact_ip):
|
||||
raise ValueError("Invalid token")
|
||||
if 0 <= port <= 65536:
|
||||
compact_port = port.to_bytes(2, 'big')
|
||||
else:
|
||||
raise TypeError(f'Invalid port: {port}')
|
||||
compact_address = compact_ip + compact_port + rpc_contact.id
|
||||
now = int(self.clock.seconds())
|
||||
originallyPublished = now - age
|
||||
self._dataStore.addPeerToBlob(rpc_contact, blob_hash, compact_address, now, originallyPublished,
|
||||
originalPublisherID)
|
||||
return b'OK'
|
||||
|
||||
@rpcmethod
|
||||
def findNode(self, rpc_contact, key):
|
||||
""" Finds a number of known nodes closest to the node/value with the
|
||||
specified key.
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
|
||||
@return: A list of contact triples closest to the specified key.
|
||||
This method will return C{k} (or C{count}, if specified)
|
||||
contacts if at all possible; it will only return fewer if the
|
||||
node is returning all of the contacts that it knows of.
|
||||
@rtype: list
|
||||
"""
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid contact id length: %i" % len(key))
|
||||
|
||||
contacts = self._routingTable.findCloseNodes(key, sender_node_id=rpc_contact.id)
|
||||
contact_triples = []
|
||||
for contact in contacts:
|
||||
contact_triples.append((contact.id, contact.address, contact.port))
|
||||
return contact_triples
|
||||
|
||||
@rpcmethod
|
||||
def findValue(self, rpc_contact, key):
|
||||
""" Return the value associated with the specified key if present in
|
||||
this node's data, otherwise execute FIND_NODE for the key
|
||||
|
||||
@param key: The hashtable key of the data to return
|
||||
@type key: str
|
||||
|
||||
@return: A dictionary containing the requested key/value pair,
|
||||
or a list of contact triples closest to the requested key.
|
||||
@rtype: dict or list
|
||||
"""
|
||||
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid blob hash length: %i" % len(key))
|
||||
|
||||
response = {
|
||||
b'token': self.make_token(rpc_contact.compact_ip()),
|
||||
}
|
||||
|
||||
if self._protocol._protocolVersion:
|
||||
response[b'protocolVersion'] = self._protocol._protocolVersion
|
||||
|
||||
# get peers we have stored for this blob
|
||||
has_other_peers = self._dataStore.hasPeersForBlob(key)
|
||||
peers = []
|
||||
if has_other_peers:
|
||||
peers.extend(self._dataStore.getPeersForBlob(key))
|
||||
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and key in self._dataStore.completed_blobs:
|
||||
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), self.externalIP.split('.'), bytearray())
|
||||
compact_port = self.peerPort.to_bytes(2, 'big')
|
||||
compact_address = compact_ip + compact_port + self.node_id
|
||||
peers.append(compact_address)
|
||||
|
||||
if peers:
|
||||
response[key] = peers
|
||||
else:
|
||||
response[b'contacts'] = self.findNode(rpc_contact, key)
|
||||
return response
|
||||
|
||||
def _generateID(self):
|
||||
""" Generates an n-bit pseudo-random identifier
|
||||
|
||||
@return: A globally unique n-bit pseudo-random identifier
|
||||
@rtype: str
|
||||
"""
|
||||
return generate_id()
|
||||
|
||||
# from lbrynet.p2p.utils import profile_deferred
|
||||
# @profile_deferred()
|
||||
@defer.inlineCallbacks
|
||||
def _iterativeFind(self, key, startupShortlist=None, rpc='findNode', exclude=None):
|
||||
""" The basic Kademlia iterative lookup operation (for nodes/values)
|
||||
|
||||
This builds a list of k "closest" contacts through iterative use of
|
||||
the "FIND_NODE" RPC, or if C{findValue} is set to C{True}, using the
|
||||
"FIND_VALUE" RPC, in which case the value (if found) may be returned
|
||||
instead of a list of contacts
|
||||
|
||||
@param key: the n-bit key (i.e. the node or value ID) to search for
|
||||
@type key: str
|
||||
@param startupShortlist: A list of contacts to use as the starting
|
||||
shortlist for this search; this is normally
|
||||
only used when the node joins the network
|
||||
@type startupShortlist: list
|
||||
@param rpc: The name of the RPC to issue to remote nodes during the
|
||||
Kademlia lookup operation (e.g. this sets whether this
|
||||
algorithm should search for a data value (if
|
||||
rpc='findValue') or not. It can thus be used to perform
|
||||
other operations that piggy-back on the basic Kademlia
|
||||
lookup operation (Entangled's "delete" RPC, for instance).
|
||||
@type rpc: str
|
||||
|
||||
@return: If C{findValue} is C{True}, the algorithm will stop as soon
|
||||
as a data value for C{key} is found, and return a dictionary
|
||||
containing the key and the found value. Otherwise, it will
|
||||
return a list of the k closest nodes to the specified key
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
|
||||
if len(key) != constants.key_bits // 8:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
|
||||
if startupShortlist is None:
|
||||
shortlist = self._routingTable.findCloseNodes(key)
|
||||
# if key != self.node_id:
|
||||
# # Update the "last accessed" timestamp for the appropriate k-bucket
|
||||
# self._routingTable.touchKBucket(key)
|
||||
if len(shortlist) == 0:
|
||||
log.warning("This node doesn't know any other nodes")
|
||||
# This node doesn't know of any other nodes
|
||||
fakeDf = defer.Deferred()
|
||||
fakeDf.callback([])
|
||||
result = yield fakeDf
|
||||
defer.returnValue(result)
|
||||
else:
|
||||
# This is used during the bootstrap process
|
||||
shortlist = startupShortlist
|
||||
|
||||
result = yield iterativeFind(self, shortlist, key, rpc, exclude=exclude)
|
||||
defer.returnValue(result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _refreshNode(self):
|
||||
""" Periodically called to perform k-bucket refreshes and data
|
||||
replication/republishing as necessary """
|
||||
yield self._refreshRoutingTable()
|
||||
self._dataStore.removeExpiredPeers()
|
||||
self._refreshStoringPeers()
|
||||
defer.returnValue(None)
|
||||
|
||||
def _refreshContacts(self):
|
||||
self._protocol._ping_queue.enqueue_maybe_ping(*self.contacts, delay=0)
|
||||
|
||||
def _refreshStoringPeers(self):
|
||||
self._protocol._ping_queue.enqueue_maybe_ping(*self._dataStore.getStoringContacts(), delay=0)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _refreshRoutingTable(self):
|
||||
nodeIDs = self._routingTable.getRefreshList(0, False)
|
||||
while nodeIDs:
|
||||
searchID = nodeIDs.pop()
|
||||
yield self.iterativeFindNode(searchID)
|
||||
defer.returnValue(None)
|
||||
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
|
||||
bottom_out_limit=20) -> typing.List['KademliaPeer']:
|
||||
accumulated: typing.List['KademliaPeer'] = []
|
||||
async with self.peer_search_junction(self.protocol.node_id, max_results=max_results,
|
||||
bottom_out_limit=bottom_out_limit) as junction:
|
||||
async for peers in junction:
|
||||
log.info("peer search: %s", peers)
|
||||
accumulated.extend(peers)
|
||||
log.info("junction done")
|
||||
log.info("context done")
|
||||
distance = Distance(node_id)
|
||||
accumulated.sort(key=lambda peer: distance(peer.node_id))
|
||||
return accumulated[:count]
|
||||
@@ -0,0 +1,203 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_valid_ipv4(address):
|
||||
try:
|
||||
ip = ipaddress.ip_address(address)
|
||||
return ip.version == 4
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
|
||||
|
||||
class PeerManager:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop):
|
||||
self._loop = loop
|
||||
self._rpc_failures: typing.Dict[
|
||||
typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
|
||||
] = {}
|
||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
self._kademlia_peers: typing.Dict[typing.Tuple[bytes, str, int], 'KademliaPeer']
|
||||
self._lock = asyncio.Lock(loop=loop)
|
||||
|
||||
async def report_failure(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
|
||||
async def report_last_sent(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
|
||||
async def report_last_replied(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
|
||||
async def report_last_requested(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
|
||||
async def clear_token(self, node_id: bytes):
|
||||
async with self._lock:
|
||||
self._node_tokens.pop(node_id, None)
|
||||
|
||||
async def update_token(self, node_id: bytes, token: bytes):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
|
||||
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
|
||||
ts, token = self._node_tokens.get(node_id, (None, None))
|
||||
if ts and ts > self._loop.time() - constants.token_secret_refresh_interval:
|
||||
return token
|
||||
|
||||
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
|
||||
return self._last_replied.get((address, udp_port))
|
||||
|
||||
def get_node_id(self, address: str, udp_port: int) -> typing.Optional[bytes]:
|
||||
return self._node_id_mapping.get((address, udp_port))
|
||||
|
||||
def get_node_address(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
return self._node_id_reverse_mapping.get(node_id)
|
||||
|
||||
async def get_node_address_and_port(self, node_id: bytes) -> typing.Optional[typing.Tuple[str, int]]:
|
||||
async with self._lock:
|
||||
addr_tuple = self._node_id_reverse_mapping.get(node_id)
|
||||
if addr_tuple and addr_tuple in self._node_id_mapping:
|
||||
return addr_tuple
|
||||
|
||||
async def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
|
||||
This is to handle peers changing addresses and ids while assuring that the we only ever have
|
||||
one node id / address tuple mapped to each other
|
||||
"""
|
||||
async with self._lock:
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
|
||||
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
|
||||
return KademliaPeer(self._loop, address, node_id, udp_port)
|
||||
|
||||
async def prune(self):
|
||||
now = self._loop.time()
|
||||
async with self._lock:
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items():
|
||||
if age < now - constants.token_secret_refresh_interval:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
|
||||
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
:return: False if peer is bad, None if peer is unknown, or True if peer is good
|
||||
"""
|
||||
|
||||
delay = self._loop.time() - constants.check_refresh_interval
|
||||
|
||||
if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
|
||||
return
|
||||
addr_tup = (address, udp_port)
|
||||
if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
|
||||
return
|
||||
previous_failure, most_recent_failure = self._rpc_failures.get((address, udp_port), (None, None))
|
||||
last_requested = self._last_requested.get((address, udp_port))
|
||||
last_replied = self._last_replied.get((address, udp_port))
|
||||
if most_recent_failure and last_replied:
|
||||
if delay < last_replied > most_recent_failure:
|
||||
return True
|
||||
elif last_replied > most_recent_failure:
|
||||
return
|
||||
return False
|
||||
elif previous_failure and most_recent_failure and most_recent_failure > delay:
|
||||
return False
|
||||
elif last_replied and last_replied > delay:
|
||||
return True
|
||||
elif last_requested and last_requested > delay:
|
||||
return None
|
||||
return
|
||||
|
||||
def peer_is_good(self, peer: 'KademliaPeer'):
|
||||
return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port)
|
||||
|
||||
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer':
|
||||
node_id, address, tcp_port = decode_compact_address(compact_address)
|
||||
return KademliaPeer(self._loop, address, node_id, tcp_port=tcp_port)
|
||||
|
||||
|
||||
class KademliaPeer:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, address: str, node_id: typing.Optional[bytes] = None,
|
||||
udp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None):
|
||||
if node_id is not None:
|
||||
if not len(node_id) == constants.hash_length:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(node_id).decode()))
|
||||
if udp_port is not None and not 0 <= udp_port <= 65536:
|
||||
raise ValueError("invalid udp port")
|
||||
if tcp_port and not 0 <= tcp_port <= 65536:
|
||||
raise ValueError("invalid tcp port")
|
||||
if not is_valid_ipv4(address):
|
||||
raise ValueError("invalid ip address")
|
||||
self.loop = loop
|
||||
self._node_id = node_id
|
||||
self.address = address
|
||||
self.udp_port = udp_port
|
||||
self.tcp_port = tcp_port
|
||||
self.protocol_version = 1
|
||||
|
||||
def update_tcp_port(self, tcp_port: int):
|
||||
self.tcp_port = tcp_port
|
||||
|
||||
def update_udp_port(self, udp_port: int):
|
||||
self.udp_port = udp_port
|
||||
|
||||
def set_id(self, node_id):
|
||||
if not self._node_id:
|
||||
self._node_id = node_id
|
||||
|
||||
@property
|
||||
def node_id(self) -> bytes:
|
||||
return self._node_id
|
||||
|
||||
def compact_address_udp(self) -> bytearray:
|
||||
return make_compact_address(self.node_id, self.address, self.udp_port)
|
||||
|
||||
def compact_address_tcp(self) -> bytearray:
|
||||
return make_compact_address(self.node_id, self.address, self.tcp_port)
|
||||
|
||||
def compact_ip(self):
|
||||
return make_compact_ip(self.address)
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, KademliaPeer):
|
||||
raise TypeError("invalid type to compare with Peer: %s" % str(type(other)))
|
||||
return (self.node_id, self.address, self.udp_port) == (other.node_id, other.address, other.udp_port)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.node_id, self.address, self.udp_port))
|
||||
@@ -1,493 +0,0 @@
|
||||
import logging
|
||||
import errno
|
||||
from binascii import hexlify
|
||||
|
||||
from twisted.internet import protocol, defer
|
||||
from lbrynet.dht import constants, encoding, msgformat, msgtypes
|
||||
from lbrynet.dht.error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PingQueue:
|
||||
"""
|
||||
Schedules a 15 minute delayed ping after a new node sends us a query. This is so the new node gets added to the
|
||||
routing table after having been given enough time for a pinhole to expire.
|
||||
"""
|
||||
|
||||
def __init__(self, node):
|
||||
self._node = node
|
||||
self._enqueued_contacts = {}
|
||||
self._pending_contacts = {}
|
||||
self._process_lc = node.get_looping_call(self._process)
|
||||
|
||||
def enqueue_maybe_ping(self, *contacts, **kwargs):
|
||||
delay = kwargs.get('delay', constants.checkRefreshInterval)
|
||||
no_op = (defer.succeed(None), lambda: None)
|
||||
for contact in contacts:
|
||||
if delay and contact not in self._enqueued_contacts:
|
||||
self._pending_contacts.setdefault(contact, self._node.clock.seconds() + delay)
|
||||
else:
|
||||
self._enqueued_contacts.setdefault(contact, no_op)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _ping(self, contact):
|
||||
if contact.contact_is_good:
|
||||
return
|
||||
try:
|
||||
yield contact.ping()
|
||||
except TimeoutError:
|
||||
pass
|
||||
except Exception as err:
|
||||
log.warning("unexpected error: %s", err)
|
||||
finally:
|
||||
if contact in self._enqueued_contacts:
|
||||
del self._enqueued_contacts[contact]
|
||||
|
||||
def _process(self):
|
||||
# move contacts that are scheduled to join the queue
|
||||
if self._pending_contacts:
|
||||
now = self._node.clock.seconds()
|
||||
for contact in [contact for contact, schedule in self._pending_contacts.items() if schedule <= now]:
|
||||
del self._pending_contacts[contact]
|
||||
self._enqueued_contacts.setdefault(contact, (defer.succeed(None), lambda: None))
|
||||
# spread pings across 60 seconds to avoid flood and/or false negatives
|
||||
step = 60.0/float(len(self._enqueued_contacts)) if self._enqueued_contacts else 0
|
||||
for index, (contact, (call, _)) in enumerate(self._enqueued_contacts.items()):
|
||||
if call.called and not contact.contact_is_good:
|
||||
self._enqueued_contacts[contact] = self._node.reactor_callLater(index*step, self._ping, contact)
|
||||
|
||||
def start(self):
|
||||
return self._node.safe_start_looping_call(self._process_lc, 60)
|
||||
|
||||
def stop(self):
|
||||
map(None, (cancel() for _, (call, cancel) in self._enqueued_contacts.items() if not call.called))
|
||||
return self._node.safe_stop_looping_call(self._process_lc)
|
||||
|
||||
|
||||
class KademliaProtocol(protocol.DatagramProtocol):
|
||||
""" Implements all low-level network-related functions of a Kademlia node """
|
||||
|
||||
msgSizeLimit = constants.udpDatagramMaxSize - 26
|
||||
|
||||
def __init__(self, node):
|
||||
self._node = node
|
||||
self._translator = msgformat.DefaultFormat()
|
||||
self._sentMessages = {}
|
||||
self._partialMessages = {}
|
||||
self._partialMessagesProgress = {}
|
||||
self._listening = defer.Deferred(None)
|
||||
self._ping_queue = PingQueue(self._node)
|
||||
self._protocolVersion = constants.protocolVersion
|
||||
self.started_listening_time = 0
|
||||
|
||||
def _migrate_incoming_rpc_args(self, contact, method, *args):
|
||||
if method == b'store' and contact.protocolVersion == 0:
|
||||
if isinstance(args[1], dict):
|
||||
blob_hash = args[0]
|
||||
token = args[1].pop(b'token', None)
|
||||
port = args[1].pop(b'port', -1)
|
||||
originalPublisherID = args[1].pop(b'lbryid', None)
|
||||
age = 0
|
||||
return (blob_hash, token, port, originalPublisherID, age), {}
|
||||
return args, {}
|
||||
|
||||
def _migrate_outgoing_rpc_args(self, contact, method, *args):
|
||||
"""
|
||||
This will reformat protocol version 0 arguments for the store function and will add the
|
||||
protocol version keyword argument to calls to contacts who will accept it
|
||||
"""
|
||||
if contact.protocolVersion == 0:
|
||||
if method == b'store':
|
||||
blob_hash, token, port, originalPublisherID, age = args
|
||||
args = (
|
||||
blob_hash, {
|
||||
b'token': token,
|
||||
b'port': port,
|
||||
b'lbryid': originalPublisherID
|
||||
}, originalPublisherID, False
|
||||
)
|
||||
return args
|
||||
return args
|
||||
if args and isinstance(args[-1], dict):
|
||||
args[-1][b'protocolVersion'] = self._protocolVersion
|
||||
return args
|
||||
return args + ({b'protocolVersion': self._protocolVersion},)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def sendRPC(self, contact, method, args):
|
||||
for _ in range(constants.rpcAttempts):
|
||||
try:
|
||||
response = yield self._sendRPC(contact, method, args)
|
||||
return response
|
||||
except TimeoutError:
|
||||
if contact.contact_is_good:
|
||||
log.debug("RETRY %s ON %s", method, contact)
|
||||
continue
|
||||
else:
|
||||
raise
|
||||
|
||||
def _sendRPC(self, contact, method, args):
|
||||
"""
|
||||
Sends an RPC to the specified contact
|
||||
|
||||
@param contact: The contact (remote node) to send the RPC to
|
||||
@type contact: kademlia.contacts.Contact
|
||||
@param method: The name of remote method to invoke
|
||||
@type method: str
|
||||
@param args: A list of (non-keyword) arguments to pass to the remote
|
||||
method, in the correct order
|
||||
@type args: tuple
|
||||
|
||||
@return: This immediately returns a deferred object, which will return
|
||||
the result of the RPC call, or raise the relevant exception
|
||||
if the remote node raised one. If C{rawResponse} is set to
|
||||
C{True}, however, it will always return the actual response
|
||||
message (which may be a C{ResponseMessage} or an
|
||||
C{ErrorMessage}).
|
||||
@rtype: twisted.internet.defer.Deferred
|
||||
"""
|
||||
msg = msgtypes.RequestMessage(self._node.node_id, method, self._migrate_outgoing_rpc_args(contact, method,
|
||||
*args))
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
|
||||
if args:
|
||||
log.debug("%s:%i SEND CALL %s(%s) TO %s:%i", self._node.externalIP, self._node.port, method,
|
||||
hexlify(args[0]), contact.address, contact.port)
|
||||
else:
|
||||
log.debug("%s:%i SEND CALL %s TO %s:%i", self._node.externalIP, self._node.port, method,
|
||||
contact.address, contact.port)
|
||||
|
||||
df = defer.Deferred()
|
||||
|
||||
def _remove_contact(failure): # remove the contact from the routing table and track the failure
|
||||
contact.update_last_failed()
|
||||
try:
|
||||
if not contact.contact_is_good:
|
||||
self._node.removeContact(contact)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
return failure
|
||||
|
||||
def _update_contact(result): # refresh the contact in the routing table
|
||||
contact.update_last_replied()
|
||||
if method == b'findValue':
|
||||
if b'token' in result:
|
||||
contact.update_token(result[b'token'])
|
||||
if b'protocolVersion' not in result:
|
||||
contact.update_protocol_version(0)
|
||||
else:
|
||||
contact.update_protocol_version(result.pop(b'protocolVersion'))
|
||||
d = self._node.addContact(contact)
|
||||
d.addCallback(lambda _: result)
|
||||
return d
|
||||
|
||||
df.addCallbacks(_update_contact, _remove_contact)
|
||||
|
||||
# Set the RPC timeout timer
|
||||
timeoutCall, cancelTimeout = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, msg.id)
|
||||
|
||||
# Transmit the data
|
||||
self._send(encodedMsg, msg.id, (contact.address, contact.port))
|
||||
self._sentMessages[msg.id] = (contact, df, timeoutCall, cancelTimeout, method, args)
|
||||
|
||||
df.addErrback(cancelTimeout)
|
||||
return df
|
||||
|
||||
def startProtocol(self):
|
||||
log.info("DHT listening on UDP %i (ext port %i)", self._node.port, self._node.externalUDPPort)
|
||||
if self._listening.called:
|
||||
self._listening = defer.Deferred()
|
||||
self._listening.callback(True)
|
||||
self.started_listening_time = self._node.clock.seconds()
|
||||
return self._ping_queue.start()
|
||||
|
||||
def datagramReceived(self, datagram, address):
|
||||
""" Handles and parses incoming RPC messages (and responses)
|
||||
|
||||
@note: This is automatically called by Twisted when the protocol
|
||||
receives a UDP datagram
|
||||
"""
|
||||
if chr(datagram[0]) == '\x00' and chr(datagram[25]) == '\x00':
|
||||
totalPackets = (datagram[1] << 8) | datagram[2]
|
||||
msgID = datagram[5:25]
|
||||
seqNumber = (datagram[3] << 8) | datagram[4]
|
||||
if msgID not in self._partialMessages:
|
||||
self._partialMessages[msgID] = {}
|
||||
self._partialMessages[msgID][seqNumber] = datagram[26:]
|
||||
if len(self._partialMessages[msgID]) == totalPackets:
|
||||
keys = self._partialMessages[msgID].keys()
|
||||
keys.sort()
|
||||
data = b''
|
||||
for key in keys:
|
||||
data += self._partialMessages[msgID][key]
|
||||
datagram = data
|
||||
del self._partialMessages[msgID]
|
||||
else:
|
||||
return
|
||||
try:
|
||||
msgPrimitive = encoding.bdecode(datagram)
|
||||
message = self._translator.fromPrimitive(msgPrimitive)
|
||||
except (encoding.DecodeError, ValueError) as err:
|
||||
# We received some rubbish here
|
||||
log.warning("Error decoding datagram %s from %s:%i - %s", hexlify(datagram),
|
||||
address[0], address[1], err)
|
||||
return
|
||||
except (IndexError, KeyError):
|
||||
log.warning("Couldn't decode dht datagram from %s", address)
|
||||
return
|
||||
|
||||
if isinstance(message, msgtypes.RequestMessage):
|
||||
# This is an RPC method request
|
||||
remoteContact = self._node.contact_manager.make_contact(message.nodeID, address[0], address[1], self)
|
||||
remoteContact.update_last_requested()
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
if remoteContact.contact_is_good is True:
|
||||
df = self._node.addContact(remoteContact)
|
||||
else:
|
||||
df = defer.succeed(None)
|
||||
df.addCallback(lambda _: self._handleRPC(remoteContact, message.id, message.request, message.args))
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
# will be added to our routing table if successful
|
||||
if remoteContact.contact_is_good is None and remoteContact.lastReplied is None:
|
||||
df.addCallback(lambda _: self._ping_queue.enqueue_maybe_ping(remoteContact))
|
||||
elif isinstance(message, msgtypes.ErrorMessage):
|
||||
# The RPC request raised a remote exception; raise it locally
|
||||
if message.exceptionType in BUILTIN_EXCEPTIONS:
|
||||
exception_type = BUILTIN_EXCEPTIONS[message.exceptionType]
|
||||
else:
|
||||
exception_type = UnknownRemoteException
|
||||
remoteException = exception_type(message.response)
|
||||
log.error("DHT RECV REMOTE EXCEPTION FROM %s:%i: %s", address[0],
|
||||
address[1], remoteException)
|
||||
if message.id in self._sentMessages:
|
||||
# Cancel timeout timer for this RPC
|
||||
remoteContact, df, timeoutCall, timeoutCanceller, method = self._sentMessages[message.id][0:5]
|
||||
timeoutCanceller()
|
||||
del self._sentMessages[message.id]
|
||||
|
||||
# reject replies coming from a different address than what we sent our request to
|
||||
if (remoteContact.address, remoteContact.port) != address:
|
||||
log.warning("Sent request to node %s at %s:%i, got reply from %s:%i",
|
||||
remoteContact.log_id(), remoteContact.address,
|
||||
remoteContact.port, address[0], address[1])
|
||||
df.errback(TimeoutError(remoteContact.id))
|
||||
return
|
||||
|
||||
# this error is returned by nodes that can be contacted but have an old
|
||||
# and broken version of the ping command, if they return it the node can
|
||||
# be contacted, so we'll treat it as a successful ping
|
||||
old_ping_error = "ping() got an unexpected keyword argument '_rpcNodeContact'"
|
||||
if isinstance(remoteException, TypeError) and \
|
||||
remoteException.message == old_ping_error:
|
||||
log.debug("old pong error")
|
||||
df.callback('pong')
|
||||
else:
|
||||
df.errback(remoteException)
|
||||
elif isinstance(message, msgtypes.ResponseMessage):
|
||||
# Find the message that triggered this response
|
||||
if message.id in self._sentMessages:
|
||||
# Cancel timeout timer for this RPC
|
||||
remoteContact, df, timeoutCall, timeoutCanceller, method = self._sentMessages[message.id][0:5]
|
||||
timeoutCanceller()
|
||||
del self._sentMessages[message.id]
|
||||
log.debug("%s:%i RECV response to %s from %s:%i", self._node.externalIP, self._node.port,
|
||||
method, remoteContact.address, remoteContact.port)
|
||||
|
||||
# When joining the network we made Contact objects for the seed nodes with node ids set to None
|
||||
# Thus, the sent_to_id will also be None, and the contact objects need the ids to be manually set.
|
||||
# These replies have be distinguished from those where the node id in the datagram does not match
|
||||
# the node id of the node we sent a message to (these messages are treated as an error)
|
||||
if remoteContact.id and remoteContact.id != message.nodeID: # sent_to_id will be None for bootstrap
|
||||
log.debug("mismatch: (%s) %s:%i (%s vs %s)", method, remoteContact.address, remoteContact.port,
|
||||
remoteContact.log_id(False), hexlify(message.nodeID))
|
||||
df.errback(TimeoutError(remoteContact.id))
|
||||
return
|
||||
elif not remoteContact.id:
|
||||
remoteContact.set_id(message.nodeID)
|
||||
|
||||
# We got a result from the RPC
|
||||
df.callback(message.response)
|
||||
else:
|
||||
# If the original message isn't found, it must have timed out
|
||||
# TODO: we should probably do something with this...
|
||||
pass
|
||||
|
||||
def _send(self, data, rpcID, address):
|
||||
""" Transmit the specified data over UDP, breaking it up into several
|
||||
packets if necessary
|
||||
|
||||
If the data is spread over multiple UDP datagrams, the packets have the
|
||||
following structure::
|
||||
| | | | | |||||||||||| 0x00 |
|
||||
|Transmission|Total number|Sequence number| RPC ID |Header end|
|
||||
| type ID | of packets |of this packet | | indicator|
|
||||
| (1 byte) | (2 bytes) | (2 bytes) |(20 bytes)| (1 byte) |
|
||||
| | | | | |||||||||||| |
|
||||
|
||||
@note: The header used for breaking up large data segments will
|
||||
possibly be moved out of the KademliaProtocol class in the
|
||||
future, into something similar to a message translator/encoder
|
||||
class (see C{kademlia.msgformat} and C{kademlia.encoding}).
|
||||
"""
|
||||
|
||||
if len(data) > self.msgSizeLimit:
|
||||
# We have to spread the data over multiple UDP datagrams,
|
||||
# and provide sequencing information
|
||||
#
|
||||
# 1st byte is transmission type id, bytes 2 & 3 are the
|
||||
# total number of packets in this transmission, bytes 4 &
|
||||
# 5 are the sequence number for this specific packet
|
||||
totalPackets = len(data) // self.msgSizeLimit
|
||||
if len(data) % self.msgSizeLimit > 0:
|
||||
totalPackets += 1
|
||||
encTotalPackets = chr(totalPackets >> 8) + chr(totalPackets & 0xff)
|
||||
seqNumber = 0
|
||||
startPos = 0
|
||||
while seqNumber < totalPackets:
|
||||
packetData = data[startPos:startPos + self.msgSizeLimit]
|
||||
encSeqNumber = chr(seqNumber >> 8) + chr(seqNumber & 0xff)
|
||||
txData = f'\x00{encTotalPackets}{encSeqNumber}{rpcID}\x00{packetData}'
|
||||
self._scheduleSendNext(txData, address)
|
||||
|
||||
startPos += self.msgSizeLimit
|
||||
seqNumber += 1
|
||||
else:
|
||||
self._scheduleSendNext(data, address)
|
||||
|
||||
def _scheduleSendNext(self, txData, address):
|
||||
"""Schedule the sending of the next UDP packet """
|
||||
delayed_call, _ = self._node.reactor_callSoon(self._write, txData, address)
|
||||
|
||||
def _write(self, txData, address):
|
||||
if self.transport:
|
||||
try:
|
||||
self.transport.write(txData, address)
|
||||
except OSError as err:
|
||||
if err.errno == errno.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
elif err.errno == errno.ENETUNREACH:
|
||||
# this should probably try to retransmit when the network connection is back
|
||||
log.error("Network is unreachable")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(txData), address[0], address[1], err.message, err.errno)
|
||||
raise err
|
||||
else:
|
||||
raise TransportNotConnected()
|
||||
|
||||
def _sendResponse(self, contact, rpcID, response):
|
||||
""" Send a RPC response to the specified contact
|
||||
"""
|
||||
msg = msgtypes.ResponseMessage(rpcID, self._node.node_id, response)
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
self._send(encodedMsg, rpcID, (contact.address, contact.port))
|
||||
|
||||
def _sendError(self, contact, rpcID, exceptionType, exceptionMessage):
|
||||
""" Send an RPC error message to the specified contact
|
||||
"""
|
||||
exceptionMessage = exceptionMessage.encode()
|
||||
msg = msgtypes.ErrorMessage(rpcID, self._node.node_id, exceptionType, exceptionMessage)
|
||||
msgPrimitive = self._translator.toPrimitive(msg)
|
||||
encodedMsg = encoding.bencode(msgPrimitive)
|
||||
self._send(encodedMsg, rpcID, (contact.address, contact.port))
|
||||
|
||||
def _handleRPC(self, senderContact, rpcID, method, args):
|
||||
""" Executes a local function in response to an RPC request """
|
||||
|
||||
# Set up the deferred callchain
|
||||
def handleError(f):
|
||||
self._sendError(senderContact, rpcID, f.type, f.getErrorMessage())
|
||||
|
||||
def handleResult(result):
|
||||
self._sendResponse(senderContact, rpcID, result)
|
||||
|
||||
df = defer.Deferred()
|
||||
df.addCallback(handleResult)
|
||||
df.addErrback(handleError)
|
||||
|
||||
# Execute the RPC
|
||||
func = getattr(self._node, method.decode(), None)
|
||||
if callable(func) and hasattr(func, "rpcmethod"):
|
||||
# Call the exposed Node method and return the result to the deferred callback chain
|
||||
# if args:
|
||||
# log.debug("%s:%i RECV CALL %s(%s) %s:%i", self._node.externalIP, self._node.port, method,
|
||||
# args[0].encode('hex'), senderContact.address, senderContact.port)
|
||||
# else:
|
||||
log.debug("%s:%i RECV CALL %s %s:%i", self._node.externalIP, self._node.port, method,
|
||||
senderContact.address, senderContact.port)
|
||||
if args and isinstance(args[-1], dict) and b'protocolVersion' in args[-1]: # args don't need reformatting
|
||||
senderContact.update_protocol_version(int(args[-1].pop(b'protocolVersion')))
|
||||
a, kw = tuple(args[:-1]), args[-1]
|
||||
else:
|
||||
senderContact.update_protocol_version(0)
|
||||
a, kw = self._migrate_incoming_rpc_args(senderContact, method, *args)
|
||||
try:
|
||||
if method != b'ping':
|
||||
result = func(senderContact, *a)
|
||||
else:
|
||||
result = func()
|
||||
except Exception as e:
|
||||
log.error("error handling request for %s:%i %s", senderContact.address, senderContact.port, method)
|
||||
df.errback(e)
|
||||
else:
|
||||
df.callback(result)
|
||||
else:
|
||||
# No such exposed method
|
||||
df.errback(AttributeError('Invalid method: %s' % method))
|
||||
return df
|
||||
|
||||
def _msgTimeout(self, messageID):
|
||||
""" Called when an RPC request message times out """
|
||||
# Find the message that timed out
|
||||
if messageID not in self._sentMessages:
|
||||
# This should never be reached
|
||||
log.error("deferred timed out, but is not present in sent messages list!")
|
||||
return
|
||||
remoteContact, df, timeout_call, timeout_canceller, method, args = self._sentMessages[messageID]
|
||||
if messageID in self._partialMessages:
|
||||
# We are still receiving this message
|
||||
self._msgTimeoutInProgress(messageID, timeout_canceller, remoteContact, df, method, args)
|
||||
return
|
||||
del self._sentMessages[messageID]
|
||||
# The message's destination node is now considered to be dead;
|
||||
# raise an (asynchronous) TimeoutError exception and update the host node
|
||||
df.errback(TimeoutError(remoteContact.id))
|
||||
|
||||
def _msgTimeoutInProgress(self, messageID, timeoutCanceller, remoteContact, df, method, args):
|
||||
# See if any progress has been made; if not, kill the message
|
||||
if self._hasProgressBeenMade(messageID):
|
||||
# Reset the RPC timeout timer
|
||||
timeoutCanceller()
|
||||
timeoutCall, cancelTimeout = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, messageID)
|
||||
self._sentMessages[messageID] = (remoteContact, df, timeoutCall, cancelTimeout, method, args)
|
||||
else:
|
||||
# No progress has been made
|
||||
if messageID in self._partialMessagesProgress:
|
||||
del self._partialMessagesProgress[messageID]
|
||||
if messageID in self._partialMessages:
|
||||
del self._partialMessages[messageID]
|
||||
df.errback(TimeoutError(remoteContact.id))
|
||||
|
||||
def _hasProgressBeenMade(self, messageID):
|
||||
return (
|
||||
messageID in self._partialMessagesProgress and
|
||||
(
|
||||
len(self._partialMessagesProgress[messageID]) !=
|
||||
len(self._partialMessages[messageID])
|
||||
)
|
||||
)
|
||||
|
||||
def stopProtocol(self):
|
||||
""" Called when the transport is disconnected.
|
||||
|
||||
Will only be called once, after all ports are disconnected.
|
||||
"""
|
||||
log.info('Stopping DHT')
|
||||
self._ping_queue.stop()
|
||||
self._node.call_later_manager.stop()
|
||||
log.info('DHT stopped')
|
||||
@@ -0,0 +1,105 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import traceback
|
||||
if typing.TYPE_CHECKING:
|
||||
from types import AsyncGeneratorType
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def cancel_task(task: typing.Optional[asyncio.Task]):
|
||||
if task and not (task.done() or task.cancelled()):
|
||||
task.cancel()
|
||||
|
||||
|
||||
def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
for task in tasks:
|
||||
cancel_task(task)
|
||||
|
||||
|
||||
def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
|
||||
while tasks:
|
||||
cancel_task(tasks.pop())
|
||||
|
||||
|
||||
class AsyncGeneratorJunction:
|
||||
"""
|
||||
A helper to interleave the results from multiple async generators into one
|
||||
async generator.
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, queue: typing.Optional[asyncio.Queue] = None):
|
||||
self.loop = loop
|
||||
self.result_queue = queue or asyncio.Queue(loop=loop)
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.running_iterators: typing.Dict[typing.AsyncGenerator, bool] = {}
|
||||
self.generator_queue: asyncio.Queue = asyncio.Queue(loop=self.loop)
|
||||
self.can_iterate = asyncio.Event(loop=self.loop)
|
||||
self.finished = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return any(self.running_iterators.values())
|
||||
|
||||
async def wait_for_generators(self):
|
||||
async def iterate(iterator: typing.AsyncGenerator):
|
||||
try:
|
||||
async for item in iterator:
|
||||
self.result_queue.put_nowait(item)
|
||||
finally:
|
||||
self.running_iterators[iterator] = False
|
||||
|
||||
while True:
|
||||
async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType'] = await self.generator_queue.get()
|
||||
self.running_iterators[async_gen] = True
|
||||
self.tasks.append(self.loop.create_task(iterate(async_gen)))
|
||||
if not self.can_iterate.is_set():
|
||||
self.can_iterate.set()
|
||||
|
||||
def add_generator(self, async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType']):
|
||||
"""
|
||||
Add an async generator. This can be called during an iteration of the generator junction.
|
||||
"""
|
||||
self.generator_queue.put_nowait(async_gen)
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
if not self.can_iterate.is_set():
|
||||
await self.can_iterate.wait()
|
||||
if not self.running:
|
||||
raise StopAsyncIteration()
|
||||
try:
|
||||
return await self.result_queue.get()
|
||||
finally:
|
||||
self.awaiting = None
|
||||
|
||||
def aclose(self):
|
||||
async def _aclose():
|
||||
for iterator in list(self.running_iterators.keys()):
|
||||
result = iterator.aclose()
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
self.running_iterators[iterator] = False
|
||||
drain_tasks(self.tasks)
|
||||
raise StopAsyncIteration()
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return self.loop.create_task(_aclose())
|
||||
|
||||
async def __aenter__(self):
|
||||
self.tasks.append(self.loop.create_task(self.wait_for_generators()))
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
try:
|
||||
await self.aclose()
|
||||
except StopAsyncIteration:
|
||||
pass
|
||||
finally:
|
||||
if exc_type:
|
||||
if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration):
|
||||
err = traceback.format_exception(exc_type, exc, tb)
|
||||
log.error(err)
|
||||
@@ -0,0 +1,76 @@
|
||||
import asyncio
|
||||
import typing
|
||||
|
||||
from lbrynet.dht import constants
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
|
||||
class DictDataStore:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager'):
|
||||
# Dictionary format:
|
||||
# { <key>: [<contact>, <value>, <lastPublished>, <originallyPublished> <original_publisher_id>] }
|
||||
self._data_store: typing.Dict[bytes,
|
||||
typing.List[typing.Tuple['KademliaPeer', bytes, float, float, bytes]]] = {}
|
||||
self._get_time = loop.time
|
||||
self._peer_manager = peer_manager
|
||||
self.completed_blobs: typing.Set[str] = set()
|
||||
|
||||
def filter_bad_and_expired_peers(self, key: bytes) -> typing.List['KademliaPeer']:
|
||||
"""
|
||||
Returns only non-expired and unknown/good peers
|
||||
"""
|
||||
peers = []
|
||||
for peer in map(lambda p: p[0],
|
||||
filter(lambda peer: self._get_time() - peer[3] < constants.data_expiration,
|
||||
self._data_store[key])):
|
||||
if self._peer_manager.peer_is_good(peer) is not False:
|
||||
peers.append(peer)
|
||||
return peers
|
||||
|
||||
def filter_expired_peers(self, key: bytes) -> typing.List['KademliaPeer']:
|
||||
"""
|
||||
Returns only non-expired peers
|
||||
"""
|
||||
return list(
|
||||
map(
|
||||
lambda p: p[0],
|
||||
filter(lambda peer: self._get_time() - peer[3] < constants.data_expiration, self._data_store[key])
|
||||
)
|
||||
)
|
||||
|
||||
def removed_expired_peers(self):
|
||||
expired_keys = []
|
||||
for key in self._data_store.keys():
|
||||
unexpired_peers = self.filter_expired_peers(key)
|
||||
if not unexpired_peers:
|
||||
expired_keys.append(key)
|
||||
else:
|
||||
self._data_store[key] = [x for x in self._data_store[key] if x[0] in unexpired_peers]
|
||||
for key in expired_keys:
|
||||
del self._data_store[key]
|
||||
|
||||
def has_peers_for_blob(self, key: bytes) -> bool:
|
||||
return key in self._data_store and len(self.filter_bad_and_expired_peers(key)) > 0
|
||||
|
||||
def add_peer_to_blob(self, contact: 'KademliaPeer', key: bytes, compact_address: bytes, last_published: int,
|
||||
originally_published: int, original_publisher_id: bytes) -> None:
|
||||
if key in self._data_store:
|
||||
if compact_address not in map(lambda store_tuple: store_tuple[1], self._data_store[key]):
|
||||
self._data_store[key].append(
|
||||
(contact, compact_address, last_published, originally_published, original_publisher_id)
|
||||
)
|
||||
else:
|
||||
self._data_store[key] = [(contact, compact_address, last_published, originally_published,
|
||||
original_publisher_id)]
|
||||
|
||||
def get_peers_for_blob(self, key: bytes) -> typing.List['KademliaPeer']:
|
||||
return [] if key not in self._data_store else [peer for peer in self.filter_bad_and_expired_peers(key)]
|
||||
|
||||
def get_storing_contacts(self) -> typing.List['KademliaPeer']:
|
||||
peers = set()
|
||||
for key in self._data_store:
|
||||
for values in self._data_store[key]:
|
||||
if values[0] not in peers:
|
||||
peers.add(values[0])
|
||||
return list(peers)
|
||||
@@ -8,20 +8,16 @@ class Distance:
|
||||
we pre-calculate the value of that point.
|
||||
"""
|
||||
|
||||
def __init__(self, key):
|
||||
if len(key) != constants.key_bits // 8:
|
||||
def __init__(self, key: bytes):
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
self.key = key
|
||||
self.val_key_one = int.from_bytes(key, 'big')
|
||||
|
||||
def __call__(self, key_two):
|
||||
def __call__(self, key_two: bytes) -> int:
|
||||
val_key_two = int.from_bytes(key_two, 'big')
|
||||
return self.val_key_one ^ val_key_two
|
||||
|
||||
def is_closer(self, a, b):
|
||||
def is_closer(self, a: bytes, b: bytes) -> bool:
|
||||
"""Returns true is `a` is closer to `key` than `b` is"""
|
||||
return self(a) < self(b)
|
||||
|
||||
def to_contact(self, contact):
|
||||
"""A convenience function for calculating the distance to a contact"""
|
||||
return self(contact.id)
|
||||
@@ -0,0 +1,381 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbrynet.utils import drain_tasks
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.error import RemoteException
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
if TYPE_CHECKING:
|
||||
from lbrynet.dht.protocol.routing_table import TreeRoutingTable
|
||||
from lbrynet.dht.protocol.protocol import KademliaProtocol
|
||||
from lbrynet.dht.peer import PeerManager, KademliaPeer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FindResponse:
|
||||
@property
|
||||
def found(self) -> bool:
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class FindNodeResponse(FindResponse):
|
||||
def __init__(self, key: bytes, close_triples: typing.List[typing.Tuple[bytes, str, int]]):
|
||||
self.key = key
|
||||
self.close_triples = close_triples
|
||||
|
||||
@property
|
||||
def found(self) -> bool:
|
||||
return self.key in [triple[0] for triple in self.close_triples]
|
||||
|
||||
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
return self.close_triples
|
||||
|
||||
|
||||
class FindValueResponse(FindResponse):
|
||||
def __init__(self, key: bytes, result_dict: typing.Dict):
|
||||
self.key = key
|
||||
self.token = result_dict[b'token']
|
||||
self.close_triples: typing.List[typing.Tuple[bytes, bytes, int]] = result_dict.get(b'contacts', [])
|
||||
self.found_compact_addresses = result_dict.get(key, [])
|
||||
|
||||
@property
|
||||
def found(self) -> bool:
|
||||
return len(self.found_compact_addresses) > 0
|
||||
|
||||
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
return [(node_id, address.decode(), port) for node_id, address, port in self.close_triples]
|
||||
|
||||
|
||||
def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']]) -> typing.List['KademliaPeer']:
|
||||
"""
|
||||
If not provided, initialize the shortlist of peers to probe to the (up to) k closest peers in the routing table
|
||||
|
||||
:param routing_table: a TreeRoutingTable
|
||||
:param key: a 48 byte hash
|
||||
:param shortlist: optional manually provided shortlist, this is done during bootstrapping when there are no
|
||||
peers in the routing table. During bootstrap the shortlist is set to be the seed nodes.
|
||||
"""
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
if not shortlist:
|
||||
shortlist = routing_table.find_close_peers(key)
|
||||
distance = Distance(key)
|
||||
shortlist.sort(key=lambda peer: distance(peer.node_id), reverse=True)
|
||||
return shortlist
|
||||
|
||||
|
||||
class IterativeFinder:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
self.loop = loop
|
||||
self.peer_manager = peer_manager
|
||||
self.routing_table = routing_table
|
||||
self.protocol = protocol
|
||||
|
||||
self.key = key
|
||||
self.bottom_out_limit = bottom_out_limit
|
||||
self.max_results = max_results
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.shortlist: typing.List['KademliaPeer'] = get_shortlist(routing_table, key, shortlist)
|
||||
self.active: typing.List['KademliaPeer'] = []
|
||||
self.contacted: typing.List[typing.Tuple[str, int]] = []
|
||||
self.distance = Distance(key)
|
||||
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None if not self.shortlist else self.shortlist[0]
|
||||
self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
|
||||
|
||||
self.iteration_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.running_probes: typing.List[asyncio.Task] = []
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.iteration_count = 0
|
||||
self.bottom_out_count = 0
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.delayed_calls: typing.List[asyncio.Handle] = []
|
||||
self.finished = asyncio.Event(loop=self.loop)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
|
||||
"""
|
||||
Send the rpc request to the peer and return an object with the FindResponse interface
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def check_result_ready(self, response: FindResponse):
|
||||
"""
|
||||
Called with a lock after adding peers from an rpc result to the shortlist.
|
||||
This method is responsible for putting a result for the generator into the Queue
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
"""
|
||||
Get an initial or cached result to be put into the Queue. Used for findValue requests where the blob
|
||||
has peers in the local data store of blobs announced to us
|
||||
"""
|
||||
return []
|
||||
|
||||
def _is_closer(self, peer: 'KademliaPeer') -> bool:
|
||||
if not self.closest_peer:
|
||||
return True
|
||||
return self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
|
||||
def _update_closest(self):
|
||||
self.shortlist.sort(key=lambda peer: self.distance(peer.node_id), reverse=True)
|
||||
if self.closest_peer and self.closest_peer is not self.shortlist[-1]:
|
||||
if self._is_closer(self.shortlist[-1]):
|
||||
self.prev_closest_peer = self.closest_peer
|
||||
self.closest_peer = self.shortlist[-1]
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
async with self.lock:
|
||||
if peer not in self.shortlist:
|
||||
self.shortlist.append(peer)
|
||||
if peer not in self.active:
|
||||
self.active.append(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
addr_tuple = (contact_triple[1], contact_triple[2])
|
||||
if addr_tuple not in self.contacted: # and not self.peer_manager.is_ignored(addr_tuple)
|
||||
found_peer = self.peer_manager.get_kademlia_peer(
|
||||
contact_triple[0], contact_triple[1], contact_triple[2]
|
||||
)
|
||||
if found_peer not in self.shortlist and self.peer_manager.peer_is_good(peer) is not False:
|
||||
self.shortlist.append(found_peer)
|
||||
self._update_closest()
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
try:
|
||||
response = await self.send_probe(peer)
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except asyncio.TimeoutError:
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
return
|
||||
except ValueError as err:
|
||||
log.warning(str(err))
|
||||
if peer in self.active:
|
||||
self.active.remove(peer)
|
||||
return
|
||||
except RemoteException:
|
||||
return
|
||||
return await self._handle_probe_result(peer, response)
|
||||
|
||||
async def _search_round(self):
|
||||
"""
|
||||
Send up to constants.alpha (5) probes to the closest peers in the shortlist
|
||||
"""
|
||||
|
||||
added = 0
|
||||
async with self.lock:
|
||||
self.shortlist.sort(key=lambda p: self.distance(p.node_id), reverse=True)
|
||||
while self.running and len(self.shortlist) and added < constants.alpha:
|
||||
peer = self.shortlist.pop()
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude or self.peer_manager.peer_is_good(peer) is False:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if (peer.address, peer.udp_port) == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
if (peer.address, peer.udp_port) not in self.contacted:
|
||||
self.contacted.append((peer.address, peer.udp_port))
|
||||
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
if t and t in self.running_probes:
|
||||
self.running_probes.remove(t)
|
||||
if not self.running_probes and self.shortlist:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.append(t)
|
||||
added += 1
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
|
||||
try:
|
||||
if self.running:
|
||||
await self._search_round()
|
||||
if self.running:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
if self.running:
|
||||
drain_tasks(self.running_probes)
|
||||
self.running = False
|
||||
|
||||
def _search(self):
|
||||
self.tasks.append(self.loop.create_task(self._search_task()))
|
||||
|
||||
def search(self):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self._search()
|
||||
|
||||
async def next_queue_or_finished(self) -> typing.List['KademliaPeer']:
|
||||
peers = self.loop.create_task(self.iteration_queue.get())
|
||||
finished = self.loop.create_task(self.finished.wait())
|
||||
err = None
|
||||
try:
|
||||
await asyncio.wait([peers, finished], loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
if peers.done():
|
||||
return peers.result()
|
||||
raise StopAsyncIteration()
|
||||
except asyncio.CancelledError as error:
|
||||
err = error
|
||||
finally:
|
||||
if not finished.done() and not finished.cancelled():
|
||||
finished.cancel()
|
||||
if not peers.done() and not peers.cancelled():
|
||||
peers.cancel()
|
||||
if err:
|
||||
raise err
|
||||
|
||||
def __aiter__(self):
|
||||
self.search()
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> typing.List['KademliaPeer']:
|
||||
try:
|
||||
if self.iteration_count == 0:
|
||||
initial_results = self.get_initial_result()
|
||||
if initial_results:
|
||||
self.iteration_queue.put_nowait(initial_results)
|
||||
result = await self.next_queue_or_finished()
|
||||
self.iteration_count += 1
|
||||
return result
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
await self.aclose()
|
||||
raise
|
||||
|
||||
def aclose(self):
|
||||
self.running = False
|
||||
|
||||
async def _aclose():
|
||||
async with self.lock:
|
||||
self.running = False
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
drain_tasks(self.tasks)
|
||||
drain_tasks(self.running_probes)
|
||||
while self.delayed_calls:
|
||||
timer = self.delayed_calls.pop()
|
||||
if timer:
|
||||
timer.cancel()
|
||||
|
||||
return asyncio.ensure_future(_aclose(), loop=self.loop)
|
||||
|
||||
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
shortlist)
|
||||
self.yielded_peers: typing.Set['KademliaPeer'] = set()
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
|
||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
def put_result(self, from_list: typing.List['KademliaPeer']):
|
||||
not_yet_yielded = [peer for peer in from_list if peer not in self.yielded_peers]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
|
||||
if to_yield:
|
||||
for peer in to_yield:
|
||||
self.yielded_peers.add(peer)
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
|
||||
def check_result_ready(self, response: FindNodeResponse):
|
||||
found = response.found and self.key != self.protocol.node_id
|
||||
|
||||
if found:
|
||||
log.info("found")
|
||||
self.put_result(self.shortlist)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
|
||||
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
|
||||
# self.bottom_out_count, self.iteration_count)
|
||||
self.bottom_out_count = 0
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
log.info("bottom out %i %i %i %i", len(self.active), len(self.contacted), len(self.shortlist),
|
||||
self.bottom_out_count)
|
||||
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
|
||||
log.info("limit hit")
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
if self.max_results and len(self.active) - len(self.yielded_peers) >= self.max_results:
|
||||
log.info("max results")
|
||||
self.put_result(self.active)
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
shortlist)
|
||||
self.blob_peers: typing.Set['KademliaPeer'] = set()
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindValueResponse:
|
||||
response = await self.protocol.get_rpc_peer(peer).find_value(self.key)
|
||||
return FindValueResponse(self.key, response)
|
||||
|
||||
def check_result_ready(self, response: FindValueResponse):
|
||||
if response.found:
|
||||
blob_peers = [self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr)
|
||||
for compact_addr in response.found_compact_addresses]
|
||||
to_yield = []
|
||||
self.bottom_out_count = 0
|
||||
for blob_peer in blob_peers:
|
||||
if blob_peer not in self.blob_peers:
|
||||
self.blob_peers.add(blob_peer)
|
||||
to_yield.append(blob_peer)
|
||||
if to_yield:
|
||||
# log.info("found %i new peers for blob", len(to_yield))
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
# if self.max_results and len(self.blob_peers) >= self.max_results:
|
||||
# log.info("enough blob peers found")
|
||||
# if not self.finished.is_set():
|
||||
# self.finished.set()
|
||||
return
|
||||
if self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
if self.bottom_out_count >= self.bottom_out_limit:
|
||||
log.info("blob peer search bottomed out")
|
||||
if not self.finished.is_set():
|
||||
self.finished.set()
|
||||
return
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
if self.protocol.data_store.has_peers_for_blob(self.key):
|
||||
return self.protocol.data_store.get_peers_for_blob(self.key)
|
||||
return []
|
||||
@@ -0,0 +1,634 @@
|
||||
import logging
|
||||
import socket
|
||||
import functools
|
||||
import hashlib
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
from asyncio.protocols import DatagramProtocol
|
||||
from asyncio.transports import DatagramTransport
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
|
||||
from lbrynet.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE
|
||||
from lbrynet.dht.error import RemoteException, TransportNotConnected
|
||||
from lbrynet.dht.protocol.routing_table import TreeRoutingTable
|
||||
from lbrynet.dht.protocol.data_store import DictDataStore
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.peer import PeerManager, KademliaPeer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
old_protocol_errors = {
|
||||
"findNode() takes exactly 2 arguments (5 given)": "0.19.1",
|
||||
"findValue() takes exactly 2 arguments (5 given)": "0.19.1"
|
||||
}
|
||||
|
||||
|
||||
class KademliaRPC:
|
||||
def __init__(self, protocol: 'KademliaProtocol', loop: asyncio.BaseEventLoop, peer_port: int = 3333):
|
||||
self.protocol = protocol
|
||||
self.loop = loop
|
||||
self.peer_port = peer_port
|
||||
self.old_token_secret: bytes = None
|
||||
self.token_secret = constants.generate_id()
|
||||
|
||||
def compact_address(self):
|
||||
compact_ip = functools.reduce(lambda buff, x: buff + bytearray([int(x)]),
|
||||
self.protocol.external_ip.split('.'), bytearray())
|
||||
compact_port = self.peer_port.to_bytes(2, 'big')
|
||||
return compact_ip + compact_port + self.protocol.node_id
|
||||
|
||||
@staticmethod
|
||||
def ping():
|
||||
return b'pong'
|
||||
|
||||
def store(self, rpc_contact: 'KademliaPeer', blob_hash: bytes, token: bytes, port: int,
|
||||
original_publisher_id: bytes, age: int) -> bytes:
|
||||
if original_publisher_id is None:
|
||||
original_publisher_id = rpc_contact.node_id
|
||||
rpc_contact.update_tcp_port(port)
|
||||
if self.loop.time() - self.protocol.started_listening_time < constants.token_secret_refresh_interval:
|
||||
pass
|
||||
elif not self.verify_token(token, rpc_contact.compact_ip()):
|
||||
raise ValueError("Invalid token")
|
||||
now = int(self.loop.time())
|
||||
originally_published = now - age
|
||||
self.protocol.data_store.add_peer_to_blob(
|
||||
rpc_contact, blob_hash, rpc_contact.compact_address_tcp(), now, originally_published, original_publisher_id
|
||||
)
|
||||
return b'OK'
|
||||
|
||||
def find_node(self, rpc_contact: 'KademliaPeer', key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid contact node_id length: %i" % len(key))
|
||||
|
||||
contacts = self.protocol.routing_table.find_close_peers(key, sender_node_id=rpc_contact.node_id)
|
||||
contact_triples = []
|
||||
for contact in contacts:
|
||||
contact_triples.append((contact.node_id, contact.address, contact.udp_port))
|
||||
return contact_triples
|
||||
|
||||
def find_value(self, rpc_contact: 'KademliaPeer', key: bytes):
|
||||
if len(key) != constants.hash_length:
|
||||
raise ValueError("invalid blob_exchange hash length: %i" % len(key))
|
||||
|
||||
response = {
|
||||
b'token': self.make_token(rpc_contact.compact_ip()),
|
||||
}
|
||||
|
||||
if self.protocol.protocol_version:
|
||||
response[b'protocolVersion'] = self.protocol.protocol_version
|
||||
|
||||
# get peers we have stored for this blob_exchange
|
||||
has_other_peers = self.protocol.data_store.has_peers_for_blob(key)
|
||||
peers = []
|
||||
if has_other_peers:
|
||||
peers.extend(self.protocol.data_store.get_peers_for_blob(key))
|
||||
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
peers.append(self.compact_address())
|
||||
if peers:
|
||||
response[key] = peers
|
||||
else:
|
||||
response[b'contacts'] = self.find_node(rpc_contact, key)
|
||||
return response
|
||||
|
||||
def refresh_token(self): # TODO: this needs to be called periodically
|
||||
self.old_token_secret = self.token_secret
|
||||
self.token_secret = constants.generate_id()
|
||||
|
||||
def make_token(self, compact_ip):
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.token_secret + compact_ip)
|
||||
return h.digest()
|
||||
|
||||
def verify_token(self, token, compact_ip):
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.token_secret + compact_ip)
|
||||
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.old_token_secret + compact_ip)
|
||||
if not token == h.digest():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class RemoteKademliaRPC:
|
||||
"""
|
||||
Encapsulates RPC calls to remote Peers
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_tracker: 'PeerManager', protocol: 'KademliaProtocol',
|
||||
peer: 'KademliaPeer'):
|
||||
self.loop = loop
|
||||
self.peer_tracker = peer_tracker
|
||||
self.protocol = protocol
|
||||
self.peer = peer
|
||||
|
||||
async def ping(self) -> bytes:
|
||||
"""
|
||||
:return: b'pong'
|
||||
"""
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_ping(self.protocol.node_id)
|
||||
)
|
||||
return response.response
|
||||
|
||||
async def store(self, blob_hash: bytes) -> bytes:
|
||||
"""
|
||||
:param blob_hash: blob hash as bytes
|
||||
:return: b'OK'
|
||||
"""
|
||||
if len(blob_hash) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid length of blob hash: {len(blob_hash)}")
|
||||
if not self.protocol.peer_port or not 0 < self.protocol.peer_port < 65535:
|
||||
raise ValueError(f"invalid tcp port: {self.protocol.peer_port}")
|
||||
token = self.peer_tracker.get_node_token(self.peer.node_id)
|
||||
if not token:
|
||||
find_value_resp = await self.find_value(blob_hash)
|
||||
token = find_value_resp[b'token']
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_store(self.protocol.node_id, blob_hash, token, self.protocol.peer_port)
|
||||
)
|
||||
return response.response
|
||||
|
||||
async def find_node(self, key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
"""
|
||||
:return: [(node_id, address, udp_port), ...]
|
||||
"""
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid length of find node key: {len(key)}")
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_node(self.protocol.node_id, key)
|
||||
)
|
||||
return [(node_id, address.decode(), udp_port) for node_id, address, udp_port in response.response]
|
||||
|
||||
async def find_value(self, key: bytes) -> typing.Union[typing.Dict]:
|
||||
"""
|
||||
:return: {
|
||||
b'token': <token bytes>,
|
||||
b'contacts': [(node_id, address, udp_port), ...]
|
||||
<key bytes>: [<blob_peer_compact_address, ...]
|
||||
}
|
||||
"""
|
||||
if len(key) != constants.hash_bits // 8:
|
||||
raise ValueError(f"invalid length of find value key: {len(key)}")
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
|
||||
)
|
||||
await self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
return response.response
|
||||
|
||||
|
||||
class PingQueue:
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, protocol: 'KademliaProtocol'):
|
||||
self._loop = loop
|
||||
self._protocol = protocol
|
||||
self._enqueued_contacts: typing.List['KademliaPeer'] = []
|
||||
self._pending_contacts: typing.Dict['KademliaPeer', float] = {}
|
||||
self._process_task: asyncio.Task = None
|
||||
self._next_task: asyncio.Future = None
|
||||
self._next_timer: asyncio.TimerHandle = None
|
||||
self._lock = asyncio.Lock()
|
||||
self._running = False
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
async def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
|
||||
delay = constants.check_refresh_interval if delay is None else delay
|
||||
async with self._lock:
|
||||
for peer in peers:
|
||||
if delay and peer not in self._enqueued_contacts:
|
||||
self._pending_contacts[peer] = self._loop.time() + delay
|
||||
elif peer not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(peer)
|
||||
if peer in self._pending_contacts:
|
||||
del self._pending_contacts[peer]
|
||||
|
||||
async def _process(self):
|
||||
async def _ping(p: 'KademliaPeer'):
|
||||
try:
|
||||
if self._protocol.peer_manager.peer_is_good(p):
|
||||
await self._protocol.add_peer(p)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(p).ping()
|
||||
except TimeoutError:
|
||||
pass
|
||||
|
||||
while True:
|
||||
tasks = []
|
||||
|
||||
async with self._lock:
|
||||
if self._enqueued_contacts or self._pending_contacts:
|
||||
now = self._loop.time()
|
||||
scheduled = [k for k, d in self._pending_contacts.items() if now >= d]
|
||||
for k in scheduled:
|
||||
del self._pending_contacts[k]
|
||||
if k not in self._enqueued_contacts:
|
||||
self._enqueued_contacts.append(k)
|
||||
while self._enqueued_contacts:
|
||||
peer = self._enqueued_contacts.pop()
|
||||
tasks.append(self._loop.create_task(_ping(peer)))
|
||||
if tasks:
|
||||
await asyncio.wait(tasks, loop=self._loop)
|
||||
|
||||
f = self._loop.create_future()
|
||||
self._loop.call_later(1.0, lambda: None if f.done() else f.set_result(None))
|
||||
await f
|
||||
|
||||
def start(self):
|
||||
assert not self._running
|
||||
self._running = True
|
||||
if not self._process_task:
|
||||
self._process_task = self._loop.create_task(self._process())
|
||||
|
||||
def stop(self):
|
||||
assert self._running
|
||||
self._running = False
|
||||
if self._process_task:
|
||||
self._process_task.cancel()
|
||||
self._process_task = None
|
||||
if self._next_task:
|
||||
self._next_task.cancel()
|
||||
self._next_task = None
|
||||
if self._next_timer:
|
||||
self._next_timer.cancel()
|
||||
self._next_timer = None
|
||||
|
||||
|
||||
class KademliaProtocol(DatagramProtocol):
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = 5.0):
|
||||
self.peer_manager = peer_manager
|
||||
self.loop = loop
|
||||
self.node_id = node_id
|
||||
self.external_ip = external_ip
|
||||
self.udp_port = udp_port
|
||||
self.peer_port = peer_port
|
||||
self.is_seed_node = False
|
||||
self.partial_messages: typing.Dict[bytes, typing.Dict[bytes, bytes]] = {}
|
||||
self.sent_messages: typing.Dict[bytes, typing.Tuple['KademliaPeer', asyncio.Future, RequestDatagram]] = {}
|
||||
self.protocol_version = constants.protocol_version
|
||||
self.started_listening_time = 0
|
||||
self.transport: DatagramTransport = None
|
||||
self.old_token_secret = constants.generate_id()
|
||||
self.token_secret = constants.generate_id()
|
||||
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id)
|
||||
self.data_store = DictDataStore(self.loop, self.peer_manager)
|
||||
self.ping_queue = PingQueue(self.loop, self)
|
||||
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
|
||||
self.lock = asyncio.Lock(loop=self.loop)
|
||||
self.rpc_timeout = rpc_timeout
|
||||
self._split_lock = asyncio.Lock(loop=self.loop)
|
||||
|
||||
def get_rpc_peer(self, peer: 'KademliaPeer') -> RemoteKademliaRPC:
|
||||
return RemoteKademliaRPC(self.loop, self.peer_manager, self, peer)
|
||||
|
||||
def stop(self):
|
||||
if self.transport:
|
||||
self.disconnect()
|
||||
|
||||
def disconnect(self):
|
||||
self.transport.close()
|
||||
|
||||
def connection_made(self, transport: DatagramTransport):
|
||||
self.transport = transport
|
||||
|
||||
def connection_lost(self, exc):
|
||||
self.stop()
|
||||
|
||||
@staticmethod
|
||||
def _migrate_incoming_rpc_args(peer: 'KademliaPeer', method: bytes, *args) -> typing.Tuple[typing.Tuple,
|
||||
typing.Dict]:
|
||||
if method == b'store' and peer.protocol_version == 0:
|
||||
if isinstance(args[1], dict):
|
||||
blob_hash = args[0]
|
||||
token = args[1].pop(b'token', None)
|
||||
port = args[1].pop(b'port', -1)
|
||||
original_publisher_id = args[1].pop(b'lbryid', None)
|
||||
age = 0
|
||||
return (blob_hash, token, port, original_publisher_id, age), {}
|
||||
return args, {}
|
||||
|
||||
async def _add_peer(self, peer: 'KademliaPeer'):
|
||||
bucket_index = self.routing_table.kbucket_index(peer.node_id)
|
||||
if self.routing_table.buckets[bucket_index].add_peer(peer):
|
||||
return True
|
||||
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
|
||||
if self.routing_table.should_split(bucket_index, peer.node_id):
|
||||
self.routing_table.split_bucket(bucket_index)
|
||||
# Retry the insertion attempt
|
||||
result = await self._add_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
return result
|
||||
else:
|
||||
# We can't split the k-bucket
|
||||
#
|
||||
# The 13 page kademlia paper specifies that the least recently contacted node in the bucket
|
||||
# shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
|
||||
# the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
|
||||
#
|
||||
# A reasonable extension to this is BEP 0005, which extends the above:
|
||||
#
|
||||
# Not all nodes that we learn about are equal. Some are "good" and some are not.
|
||||
# Many nodes using the DHT are able to send queries and receive responses,
|
||||
# but are not able to respond to queries from other nodes. It is important that
|
||||
# each node's routing table must contain only known good nodes. A good node is
|
||||
# a node has responded to one of our queries within the last 15 minutes. A node
|
||||
# is also good if it has ever responded to one of our queries and has sent us a
|
||||
# query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
|
||||
# questionable. Nodes become bad when they fail to respond to multiple queries
|
||||
# in a row. Nodes that we know are good are given priority over nodes with unknown status.
|
||||
#
|
||||
# When there are bad or questionable nodes in the bucket, the least recent is selected for
|
||||
# potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
|
||||
# contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
|
||||
# is ignored if the pinged node replies.
|
||||
|
||||
not_good_contacts = self.routing_table.buckets[bucket_index].get_bad_or_unknown_peers()
|
||||
not_recently_replied = []
|
||||
for p in not_good_contacts:
|
||||
last_replied = self.peer_manager.get_last_replied(p.address, p.udp_port)
|
||||
if not last_replied or last_replied + 60 < self.loop.time():
|
||||
not_recently_replied.append(p)
|
||||
if not_recently_replied:
|
||||
to_replace = not_recently_replied[0]
|
||||
else:
|
||||
to_replace = self.routing_table.buckets[bucket_index].peers[0]
|
||||
last_replied = self.peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
|
||||
if last_replied and last_replied + 60 > self.loop.time():
|
||||
return False
|
||||
log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
|
||||
try:
|
||||
to_replace_rpc = self.get_rpc_peer(to_replace)
|
||||
await to_replace_rpc.ping()
|
||||
return False
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
|
||||
to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
|
||||
if to_replace in self.routing_table.buckets[bucket_index]:
|
||||
self.routing_table.buckets[bucket_index].remove_peer(to_replace)
|
||||
return await self._add_peer(peer)
|
||||
|
||||
async def add_peer(self, peer: 'KademliaPeer') -> bool:
|
||||
if peer.node_id == self.node_id:
|
||||
return False
|
||||
async with self._split_lock:
|
||||
return await self._add_peer(peer)
|
||||
|
||||
async def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
|
||||
assert sender_contact.node_id != self.node_id, (binascii.hexlify(sender_contact.node_id)[:8].decode(),
|
||||
binascii.hexlify(self.node_id)[:8].decode())
|
||||
method = message.method
|
||||
if method not in [b'ping', b'store', b'findNode', b'findValue']:
|
||||
raise AttributeError('Invalid method: %s' % message.method.decode())
|
||||
if message.args and isinstance(message.args[-1], dict) and b'protocolVersion' in message.args[-1]:
|
||||
# args don't need reformatting
|
||||
a, kw = tuple(message.args[:-1]), message.args[-1]
|
||||
else:
|
||||
a, kw = self._migrate_incoming_rpc_args(sender_contact, message.method, *message.args)
|
||||
log.debug("%s:%i RECV CALL %s %s:%i", self.external_ip, self.udp_port, message.method.decode(),
|
||||
sender_contact.address, sender_contact.udp_port)
|
||||
|
||||
if method == b'ping':
|
||||
result = self.node_rpc.ping()
|
||||
elif method == b'store':
|
||||
blob_hash, token, port, original_publisher_id, age = a
|
||||
result = self.node_rpc.store(sender_contact, blob_hash, token, port, original_publisher_id, age)
|
||||
elif method == b'findNode':
|
||||
key, = a
|
||||
result = self.node_rpc.find_node(sender_contact, key)
|
||||
else:
|
||||
assert method == b'findValue'
|
||||
key, = a
|
||||
result = self.node_rpc.find_value(sender_contact, key)
|
||||
|
||||
await self.send_response(
|
||||
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
|
||||
)
|
||||
|
||||
async def handle_request_datagram(self, address, request_datagram: RequestDatagram):
|
||||
# This is an RPC method request
|
||||
await self.peer_manager.report_last_requested(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(request_datagram.node_id, address[0], address[1])
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
await self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
# will be added to our routing table if successful
|
||||
is_good = self.peer_manager.peer_is_good(peer)
|
||||
if is_good is None:
|
||||
await self.ping_queue.enqueue_maybe_ping(peer)
|
||||
elif is_good is True:
|
||||
await self.add_peer(peer)
|
||||
|
||||
except Exception as err:
|
||||
log.warning("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
str(err))
|
||||
await self.send_error(
|
||||
peer,
|
||||
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
|
||||
str(err).encode())
|
||||
)
|
||||
|
||||
async def handle_response_datagram(self, address: typing.Tuple[str, int], response_datagram: ResponseDatagram):
|
||||
# Find the message that triggered this response
|
||||
if response_datagram.rpc_id in self.sent_messages:
|
||||
peer, df, request = self.sent_messages[response_datagram.rpc_id]
|
||||
if peer.address != address[0]:
|
||||
df.set_exception(RemoteException(
|
||||
f"response from {address[0]}:{address[1]}, "
|
||||
f"expected {peer.address}:{peer.udp_port}")
|
||||
)
|
||||
return
|
||||
peer.set_id(response_datagram.node_id)
|
||||
# We got a result from the RPC
|
||||
if peer.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("node has our node id"))
|
||||
return
|
||||
elif response_datagram.node_id == self.node_id:
|
||||
df.set_exception(RemoteException("incoming message is from our node id"))
|
||||
return
|
||||
await self.peer_manager.report_last_replied(address[0], address[1])
|
||||
await self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not df.cancelled():
|
||||
df.set_result(response_datagram)
|
||||
await self.add_peer(peer)
|
||||
else:
|
||||
log.warning("%s:%i replied, but after we cancelled the request attempt",
|
||||
peer.address, peer.udp_port)
|
||||
else:
|
||||
# If the original message isn't found, it must have timed out
|
||||
# TODO: we should probably do something with this...
|
||||
pass
|
||||
|
||||
def handle_error_datagram(self, address, error_datagram: ErrorDatagram):
|
||||
# The RPC request raised a remote exception; raise it locally
|
||||
remote_exception = RemoteException(f"{error_datagram.exception_type}({error_datagram.response})")
|
||||
if error_datagram.rpc_id in self.sent_messages:
|
||||
peer, df, request = self.sent_messages.pop(error_datagram.rpc_id)
|
||||
|
||||
error_msg = f"" \
|
||||
f"Error sending '{request.method}' to {peer.address}:{peer.udp_port}\n" \
|
||||
f"Args: {request.args}\n" \
|
||||
f"Raised: {str(remote_exception)}"
|
||||
if error_datagram.response not in old_protocol_errors:
|
||||
log.warning(error_msg)
|
||||
else:
|
||||
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
peer.address, peer.udp_port, old_protocol_errors[error_datagram.response])
|
||||
|
||||
# reject replies coming from a different address than what we sent our request to
|
||||
if (peer.address, peer.udp_port) != address:
|
||||
log.error("node id mismatch in reply")
|
||||
remote_exception = TimeoutError(peer.node_id)
|
||||
df.set_exception(remote_exception)
|
||||
return
|
||||
else:
|
||||
if error_datagram.response not in old_protocol_errors:
|
||||
msg = f"Received error from {address[0]}:{address[1]}, but it isn't in response to a " \
|
||||
f"pending request: {str(remote_exception)}"
|
||||
log.warning(msg)
|
||||
else:
|
||||
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
address[0], address[1], old_protocol_errors[error_datagram.response])
|
||||
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None:
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
self.loop.create_task(self.peer_manager.report_failure(address[0], address[1]))
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
return
|
||||
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.loop.create_task(self.handle_request_datagram(address, message))
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
self.handle_error_datagram(address, message)
|
||||
else:
|
||||
assert isinstance(message, ResponseDatagram), "sanity"
|
||||
self.loop.create_task(self.handle_response_datagram(address, message))
|
||||
|
||||
async def send_request(self, peer: 'KademliaPeer', request: RequestDatagram) -> ResponseDatagram:
|
||||
await self._send(peer, request)
|
||||
response_fut = self.sent_messages[request.rpc_id][1]
|
||||
try:
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
await self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
return response
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
self.routing_table.remove_peer(peer)
|
||||
raise
|
||||
|
||||
async def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
|
||||
await self._send(peer, response)
|
||||
|
||||
async def send_error(self, peer: 'KademliaPeer', error: ErrorDatagram):
|
||||
await self._send(peer, error)
|
||||
|
||||
async def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
|
||||
ErrorDatagram]):
|
||||
if not self.transport:
|
||||
raise TransportNotConnected()
|
||||
|
||||
data = message.bencode()
|
||||
if len(data) > constants.msg_size_limit:
|
||||
log.exception("unexpected: %i vs %i", len(data), constants.msg_size_limit)
|
||||
raise ValueError()
|
||||
if isinstance(message, (RequestDatagram, ResponseDatagram)):
|
||||
assert message.node_id == self.node_id, message
|
||||
if isinstance(message, RequestDatagram):
|
||||
assert self.node_id != peer.node_id
|
||||
|
||||
def pop_from_sent_messages(_):
|
||||
if message.rpc_id in self.sent_messages:
|
||||
self.sent_messages.pop(message.rpc_id)
|
||||
|
||||
async with self.lock:
|
||||
if isinstance(message, RequestDatagram):
|
||||
response_fut = self.loop.create_future()
|
||||
response_fut.add_done_callback(pop_from_sent_messages)
|
||||
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
|
||||
try:
|
||||
self.transport.sendto(data, (peer.address, peer.udp_port))
|
||||
except OSError as err:
|
||||
# TODO: handle ENETUNREACH
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), peer.address, peer.udp_port, str(err), err.errno)
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.sent_messages[message.rpc_id][1].set_exception(err)
|
||||
else:
|
||||
raise err
|
||||
if isinstance(message, RequestDatagram):
|
||||
await self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
await self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
|
||||
def change_token(self):
|
||||
self.old_token_secret = self.token_secret
|
||||
self.token_secret = constants.generate_id()
|
||||
|
||||
def make_token(self, compact_ip):
|
||||
return constants.digest(self.token_secret + compact_ip)
|
||||
|
||||
def verify_token(self, token, compact_ip):
|
||||
h = constants.hash_class()
|
||||
h.update(self.token_secret + compact_ip)
|
||||
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
|
||||
h = constants.hash_class()
|
||||
h.update(self.old_token_secret + compact_ip)
|
||||
if not token == h.digest():
|
||||
return False
|
||||
return True
|
||||
|
||||
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer') -> typing.Tuple[bytes, bool]:
|
||||
try:
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.info("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, True
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s" % err)
|
||||
except Exception as err:
|
||||
if 'Invalid token' in str(err):
|
||||
await self.peer_manager.clear_token(peer.node_id)
|
||||
else:
|
||||
log.exception("Unexpected error while storing blob_hash")
|
||||
return peer.node_id, False
|
||||
|
||||
def _write(self, data: bytes, address: typing.Tuple[str, int]):
|
||||
if self.transport:
|
||||
try:
|
||||
self.transport.sendto(data, address)
|
||||
except OSError as err:
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
# elif err.errno == socket.ENETUNREACH:
|
||||
# # this should probably try to retransmit when the network connection is back
|
||||
# log.error("Network is unreachable")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), address[0], address[1], str(err), err.errno)
|
||||
raise err
|
||||
else:
|
||||
raise TransportNotConnected()
|
||||
@@ -0,0 +1,305 @@
|
||||
import asyncio
|
||||
import random
|
||||
import logging
|
||||
import typing
|
||||
import itertools
|
||||
|
||||
from lbrynet.dht import constants
|
||||
from lbrynet.dht.protocol.distance import Distance
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbrynet.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KBucket:
|
||||
""" Description - later
|
||||
"""
|
||||
|
||||
def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int, node_id: bytes):
|
||||
"""
|
||||
@param range_min: The lower boundary for the range in the n-bit ID
|
||||
space covered by this k-bucket
|
||||
@param range_max: The upper boundary for the range in the ID space
|
||||
covered by this k-bucket
|
||||
"""
|
||||
self._peer_manager = peer_manager
|
||||
self.last_accessed = 0
|
||||
self.range_min = range_min
|
||||
self.range_max = range_max
|
||||
self.peers: typing.List['KademliaPeer'] = []
|
||||
self._node_id = node_id
|
||||
|
||||
def add_peer(self, peer: 'KademliaPeer') -> bool:
|
||||
""" Add contact to _contact list in the right order. This will move the
|
||||
contact to the end of the k-bucket if it is already present.
|
||||
|
||||
@raise kademlia.kbucket.BucketFull: Raised when the bucket is full and
|
||||
the contact isn't in the bucket
|
||||
already
|
||||
|
||||
@param peer: The contact to add
|
||||
@type peer: dht.contact._Contact
|
||||
"""
|
||||
if peer in self.peers:
|
||||
# Move the existing contact to the end of the list
|
||||
# - using the new contact to allow add-on data
|
||||
# (e.g. optimization-specific stuff) to pe updated as well
|
||||
self.peers.remove(peer)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
elif len(self.peers) < constants.k:
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
# raise BucketFull("No space in bucket to insert contact")
|
||||
|
||||
def get_peer(self, node_id: bytes) -> 'KademliaPeer':
|
||||
for peer in self.peers:
|
||||
if peer.node_id == node_id:
|
||||
return peer
|
||||
raise IndexError(node_id)
|
||||
|
||||
def get_peers(self, count=-1, exclude_contact=None, sort_distance_to=None) -> typing.List['KademliaPeer']:
|
||||
""" Returns a list containing up to the first count number of contacts
|
||||
|
||||
@param count: The amount of contacts to return (if 0 or less, return
|
||||
all contacts)
|
||||
@type count: int
|
||||
@param exclude_contact: A node node_id to exclude; if this contact is in
|
||||
the list of returned values, it will be
|
||||
discarded before returning. If a C{str} is
|
||||
passed as this argument, it must be the
|
||||
contact's ID.
|
||||
@type exclude_contact: str
|
||||
|
||||
@param sort_distance_to: Sort distance to the node_id, defaulting to the parent node node_id. If False don't
|
||||
sort the contacts
|
||||
|
||||
@raise IndexError: If the number of requested contacts is too large
|
||||
|
||||
@return: Return up to the first count number of contacts in a list
|
||||
If no contacts are present an empty is returned
|
||||
@rtype: list
|
||||
"""
|
||||
peers = [peer for peer in self.peers if peer.node_id != exclude_contact]
|
||||
|
||||
# Return all contacts in bucket
|
||||
if count <= 0:
|
||||
count = len(peers)
|
||||
|
||||
# Get current contact number
|
||||
current_len = len(peers)
|
||||
|
||||
# If count greater than k - return only k contacts
|
||||
if count > constants.k:
|
||||
count = constants.k
|
||||
|
||||
if not current_len:
|
||||
return peers
|
||||
|
||||
if sort_distance_to is False:
|
||||
pass
|
||||
else:
|
||||
sort_distance_to = sort_distance_to or self._node_id
|
||||
peers.sort(key=lambda c: Distance(sort_distance_to)(c.node_id))
|
||||
|
||||
return peers[:min(current_len, count)]
|
||||
|
||||
def get_bad_or_unknown_peers(self) -> typing.List['KademliaPeer']:
|
||||
peer = self.get_peers(sort_distance_to=False)
|
||||
return [
|
||||
peer for peer in peer
|
||||
if self._peer_manager.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port) is not True
|
||||
]
|
||||
|
||||
def remove_peer(self, peer: 'KademliaPeer') -> None:
|
||||
self.peers.remove(peer)
|
||||
|
||||
def key_in_range(self, key: bytes) -> bool:
|
||||
""" Tests whether the specified key (i.e. node ID) is in the range
|
||||
of the n-bit ID space covered by this k-bucket (in otherwords, it
|
||||
returns whether or not the specified key should be placed in this
|
||||
k-bucket)
|
||||
|
||||
@param key: The key to test
|
||||
@type key: str or int
|
||||
|
||||
@return: C{True} if the key is in this k-bucket's range, or C{False}
|
||||
if not.
|
||||
@rtype: bool
|
||||
"""
|
||||
return self.range_min <= int.from_bytes(key, 'big') < self.range_max
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.peers)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.peers
|
||||
|
||||
|
||||
class TreeRoutingTable:
|
||||
""" This class implements a routing table used by a Node class.
|
||||
|
||||
The Kademlia routing table is a binary tree whose leaves are k-buckets,
|
||||
where each k-bucket contains nodes with some common prefix of their IDs.
|
||||
This prefix is the k-bucket's position in the binary tree; it therefore
|
||||
covers some range of ID values, and together all of the k-buckets cover
|
||||
the entire n-bit ID (or key) space (with no overlap).
|
||||
|
||||
@note: In this implementation, nodes in the tree (the k-buckets) are
|
||||
added dynamically, as needed; this technique is described in the 13-page
|
||||
version of the Kademlia paper, in section 2.4. It does, however, use the
|
||||
ping RPC-based k-bucket eviction algorithm described in section 2.2 of
|
||||
that paper.
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes):
|
||||
self._loop = loop
|
||||
self._peer_manager = peer_manager
|
||||
self._parent_node_id = parent_node_id
|
||||
self.buckets: typing.List[KBucket] = [
|
||||
KBucket(
|
||||
self._peer_manager, range_min=0, range_max=2 ** constants.hash_bits, node_id=self._parent_node_id
|
||||
)
|
||||
]
|
||||
|
||||
def get_peers(self) -> typing.List['KademliaPeer']:
|
||||
return list(itertools.chain.from_iterable(map(lambda bucket: bucket.peers, self.buckets)))
|
||||
|
||||
def should_split(self, bucket_index: int, to_add: bytes) -> bool:
|
||||
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
|
||||
if self.buckets[bucket_index].key_in_range(self._parent_node_id):
|
||||
return True
|
||||
contacts = self.get_peers()
|
||||
distance = Distance(self._parent_node_id)
|
||||
contacts.sort(key=lambda c: distance(c.node_id))
|
||||
kth_contact = contacts[-1] if len(contacts) < constants.k else contacts[constants.k - 1]
|
||||
return distance(to_add) < distance(kth_contact.node_id)
|
||||
|
||||
def find_close_peers(self, key: bytes, count: typing.Optional[int] = None,
|
||||
sender_node_id: typing.Optional[bytes] = None) -> typing.List['KademliaPeer']:
|
||||
exclude = [self._parent_node_id]
|
||||
if sender_node_id:
|
||||
exclude.append(sender_node_id)
|
||||
if key in exclude:
|
||||
exclude.remove(key)
|
||||
count = count or constants.k
|
||||
distance = Distance(key)
|
||||
contacts = self.get_peers()
|
||||
contacts = [c for c in contacts if c.node_id not in exclude]
|
||||
if contacts:
|
||||
contacts.sort(key=lambda c: distance(c.node_id))
|
||||
return contacts[:min(count, len(contacts))]
|
||||
return []
|
||||
|
||||
def get_peer(self, contact_id: bytes) -> 'KademliaPeer':
|
||||
"""
|
||||
@raise IndexError: No contact with the specified contact ID is known
|
||||
by this node
|
||||
"""
|
||||
return self.buckets[self.kbucket_index(contact_id)].get_peer(contact_id)
|
||||
|
||||
def get_refresh_list(self, start_index: int = 0, force: bool = False) -> typing.List[bytes]:
|
||||
bucket_index = start_index
|
||||
refresh_ids = []
|
||||
now = int(self._loop.time())
|
||||
for bucket in self.buckets[start_index:]:
|
||||
if force or now - bucket.last_accessed >= constants.refresh_interval:
|
||||
to_search = self.midpoint_id_in_bucket_range(bucket_index)
|
||||
refresh_ids.append(to_search)
|
||||
bucket_index += 1
|
||||
return refresh_ids
|
||||
|
||||
def remove_peer(self, peer: 'KademliaPeer') -> None:
|
||||
if not peer.node_id:
|
||||
return
|
||||
bucket_index = self.kbucket_index(peer.node_id)
|
||||
try:
|
||||
self.buckets[bucket_index].remove_peer(peer)
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
def touch_kbucket(self, key: bytes) -> None:
|
||||
self.touch_kbucket_by_index(self.kbucket_index(key))
|
||||
|
||||
def touch_kbucket_by_index(self, bucket_index: int):
|
||||
self.buckets[bucket_index].last_accessed = int(self._loop.time())
|
||||
|
||||
def kbucket_index(self, key: bytes) -> int:
|
||||
i = 0
|
||||
for bucket in self.buckets:
|
||||
if bucket.key_in_range(key):
|
||||
return i
|
||||
else:
|
||||
i += 1
|
||||
return i
|
||||
|
||||
def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
|
||||
return random_id.to_bytes(constants.hash_length, 'big')
|
||||
|
||||
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
|
||||
return int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
|
||||
|
||||
def split_bucket(self, old_bucket_index: int) -> None:
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
cover the same range in the key/ID space
|
||||
|
||||
@param old_bucket_index: The index of k-bucket to split (in this table's
|
||||
list of k-buckets)
|
||||
@type old_bucket_index: int
|
||||
"""
|
||||
# Resize the range of the current (old) k-bucket
|
||||
old_bucket = self.buckets[old_bucket_index]
|
||||
split_point = old_bucket.range_max - (old_bucket.range_max - old_bucket.range_min) // 2
|
||||
# Create a new k-bucket to cover the range split off from the old bucket
|
||||
new_bucket = KBucket(self._peer_manager, split_point, old_bucket.range_max, self._parent_node_id)
|
||||
old_bucket.range_max = split_point
|
||||
# Now, add the new bucket into the routing table tree
|
||||
self.buckets.insert(old_bucket_index + 1, new_bucket)
|
||||
# Finally, copy all nodes that belong to the new k-bucket into it...
|
||||
for contact in old_bucket.peers:
|
||||
if new_bucket.key_in_range(contact.node_id):
|
||||
new_bucket.add_peer(contact)
|
||||
# ...and remove them from the old bucket
|
||||
for contact in new_bucket.peers:
|
||||
old_bucket.remove_peer(contact)
|
||||
|
||||
def join_buckets(self):
|
||||
to_pop = [i for i, bucket in enumerate(self.buckets) if not len(bucket)]
|
||||
if not to_pop:
|
||||
return
|
||||
log.info("join buckets %i", len(to_pop))
|
||||
bucket_index_to_pop = to_pop[0]
|
||||
assert len(self.buckets[bucket_index_to_pop]) == 0
|
||||
can_go_lower = bucket_index_to_pop - 1 >= 0
|
||||
can_go_higher = bucket_index_to_pop + 1 < len(self.buckets)
|
||||
assert can_go_higher or can_go_lower
|
||||
bucket = self.buckets[bucket_index_to_pop]
|
||||
if can_go_lower and can_go_higher:
|
||||
midpoint = ((bucket.range_max - bucket.range_min) // 2) + bucket.range_min
|
||||
self.buckets[bucket_index_to_pop - 1].range_max = midpoint - 1
|
||||
self.buckets[bucket_index_to_pop + 1].range_min = midpoint
|
||||
elif can_go_lower:
|
||||
self.buckets[bucket_index_to_pop - 1].range_max = bucket.range_max
|
||||
elif can_go_higher:
|
||||
self.buckets[bucket_index_to_pop + 1].range_min = bucket.range_min
|
||||
self.buckets.remove(bucket)
|
||||
return self.join_buckets()
|
||||
|
||||
def contact_in_routing_table(self, address_tuple: typing.Tuple[str, int]) -> bool:
|
||||
for bucket in self.buckets:
|
||||
for contact in bucket.get_peers(sort_distance_to=False):
|
||||
if address_tuple[0] == contact.address and address_tuple[1] == contact.udp_port:
|
||||
return True
|
||||
return False
|
||||
|
||||
def buckets_with_contacts(self) -> int:
|
||||
count = 0
|
||||
for bucket in self.buckets:
|
||||
if len(bucket):
|
||||
count += 1
|
||||
return count
|
||||