Compare commits

..
5 Commits
Author SHA1 Message Date
Jack Robison 41b9ad9458 Bump version 0.19.3rc2 --> 0.19.3rc3
Signed-off-by: Jack Robison <jackrobison@lbry.io>
2018-05-02 15:51:04 -04:00
Jack Robison 59138f8579 version 2018-05-02 15:50:45 -04:00
Jack Robison 40995f2809 download headers from s3 when more than 10 chunks behind 2018-05-02 15:35:17 -04:00
Jack Robison 1de6966cd2 fix tests 2018-05-02 15:31:33 -04:00
Alex Grintsvayg 00b8166bb1 undeprecated report_bug while app team works on a new bug reporting flow 2018-05-02 15:12:40 -04:00
407 changed files with 27871 additions and 29498 deletions

No files matched your search

+44
View File
@@ -0,0 +1,44 @@
version: 1.0.{build}
environment:
GH_TOKEN:
secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
key_pass:
secure: u6DydPcdrUJlxGL9uc7yQRYG8+5rY6aAEE9nfCSzFyNzZlX9NniOp8Uh5ZKQqX7bGEngLI6ipbLfiJvn0XFnhbn2iTkOuMqOXVJVOehvwlQ=
pfx_key:
secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg=
AWS_ACCESS_KEY_ID:
secure: iVGwoJ7ogspjSmuqr+haVPLglSgQsp6tUZx6mIlKH7Q=
AWS_SECRET_ACCESS_KEY:
secure: zKaqdZGPl0exDL5YhJkb33prSemC9Rzg9S7Lw2wFy1WnJ6ffgl6mQH7jqJDUTqsY
#notifications:
# - provider: Slack
# incoming_webhook:
# secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw=
clone_folder: c:\projects\lbry
test_script:
- cd C:\projects\lbry\
- pip install cython
- pip install mock pylint unqlite
- pip install .
- pylint lbrynet
# disable tests for now so that appveyor can build the app
- set PYTHONPATH=.
- trial lbrynet.tests
# TODO: integration tests do not work
#- python -m unittest discover tests/integration
#- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
build_script:
- cd C:\projects\lbry\build\
- ps: .\build.ps1
artifacts:
- path: build\dist\*.zip
name: lbrynet-daemon
-7
View File
@@ -1,7 +0,0 @@
.git
.tox
__pycache__
dist
lbrynet.egg-info
docs
tests
+1 -22
View File
@@ -4,7 +4,7 @@ Thanks for reporting an issue to LBRY and helping us improve!
To make it possible for us to help you, please fill out below information carefully.
Before reporting any issues, please make sure that you're using the latest version.
- App: https://github.com/lbryio/lbry-desktop/releases
- App: https://github.com/lbryio/lbry-app/releases
- Daemon: https://github.com/lbryio/lbry/releases
We are also available on Discord at https://chat.lbry.io
@@ -51,24 +51,3 @@ Tell us what happens instead
## Screenshots
<!-- If a screenshot would help explain the bug, please include one or two here -->
## Internal Use
### Acceptance Criteria
1.
2.
3.
### Definition of Done
- [ ] Tested against acceptance criteria
- [ ] Tested against the assumptions of user story
- [ ] The project builds without errors
- [ ] Unit tests are written and passing
- [ ] Tests on devices/browsers listed in the issue have passed
- [ ] QA performed & issues resolved
- [ ] Refactoring completed
- [ ] Any configuration or build changes documented
- [ ] Documentation updated
- [ ] Peer Code Review performed
-38
View File
@@ -1,38 +0,0 @@
## PR Checklist
Please check all that apply to this PR using "x":
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
- [ ] I have checked that this PR does not introduce a breaking change
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
## PR Type
What kind of change does this PR introduce?
Why is this change necessary?
<!-- Please check all that apply to this PR using "x". -->
- [ ] Bugfix
- [ ] Feature
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
- [ ] Code style update (formatting)
- [ ] Refactoring (no functional changes)
- [ ] Documentation changes
- [ ] Other - Please describe:
## Fixes
Issue Number: N/A
## What is the current behavior?
## What is the new behavior?
## Other information
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
+23 -7
View File
@@ -1,9 +1,25 @@
/build
/dist
/.tox
/.idea
/.coverage
*.pyc
*.egg
*.so
*.xml
*.iml
*.log
*.pem
*.decTest
*.prof
.#*
lbrynet.egg-info
__pycache__
/build/build
/build/dist
/bulid/requirements_base.txt
/lbrynet.egg-info
/docs_build
/lbry-venv
.idea/
.coverage
.DS_Store
# temporary files from the twisted.trial test runner
_trial_temp/
+7 -11
View File
@@ -9,7 +9,7 @@
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS,schema
ignore=CVS
# Add files or directories matching the regex patterns to the
# blacklist. The regex matches against base names, not paths.
@@ -62,7 +62,7 @@ confidence=
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once).You can also use "--disable=all" to
# disable everything first and then re-enable specific checks. For example, if
# disable everything first and then reenable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
@@ -121,11 +121,7 @@ disable=
unidiomatic-typecheck,
global-at-module-level,
inconsistent-return-statements,
keyword-arg-before-vararg,
assignment-from-no-return,
useless-return,
assignment-from-none,
stop-iteration-return
keyword-arg-before-vararg
[REPORTS]
@@ -354,13 +350,13 @@ ignore-mixin-members=yes
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=leveldb,distutils
ignored-modules=twisted.internet.reactor,leveldb,distutils
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set). This supports can work
# with qualified names.
# ignored-classes=
ignored-classes=twisted.internet.reactor,RequestMessage
# List of members which are set dynamically and missed by pylint inference
@@ -390,7 +386,7 @@ int-import-graph=
[DESIGN]
# Maximum number of arguments for function / method
max-args=10
max-args=5
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
@@ -409,7 +405,7 @@ max-branches=12
max-statements=50
# Maximum number of parents for a class (see R0901).
max-parents=8
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=7
+39 -132
View File
@@ -1,140 +1,47 @@
sudo: required
dist: xenial
os: linux
dist: trusty
language: python
python: "3.7"
python: 2.7
jobs:
include:
branches:
except:
- gh-pages
- stage: code quality
name: "pylint lbrynet"
install:
- pip install astroid==2.0.4 aiohttp==3.4.4
# newer astroid and aiohttp fails in pylint so we pre-install older version
- pip install pylint
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .
script: pylint lbrynet
- stage: test
name: "Unit Tests"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .
script:
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
after_success:
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- name: "Integration Tests"
install:
- pip install tox-travis coverage
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
script: tox
after_success:
- coverage combine tests/
- bash <(curl -s https://codecov.io/bash)
- stage: build
name: "Windows"
language: generic
services:
- docker
install:
- docker pull lbry/pyinstaller34_32bits:py371
script:
- python scripts/set_build.py
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
file: dist/lbrynet-windows.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
paths:
- lbrynet-windows.zip
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- &build
name: "Linux"
env: OS=linux
install:
- pip3 install pyinstaller
- pip3 install git+https://github.com/lbryio/torba.git
- python scripts/set_build.py
- pip3 install -e .
script:
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
- chmod +x dist/lbrynet
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
- ./dist/lbrynet --version
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
file: dist/lbrynet-${OS}.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
paths:
- lbrynet-${OS}.zip
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- <<: *build
name: "Mac"
os: osx
osx_image: xcode8.3
language: generic
env: OS=mac
cache: false
before_install:
- brew upgrade python || true
- brew upgrade python || true
install:
- python3 --version
- pip3 --version
- pip3 install pyinstaller
- git clone https://github.com/lbryio/torba.git --depth 1
- sed -i -e "s/'plyvel',//" torba/setup.py
- cd torba && pip3 install -e . && cd ..
- python3 scripts/set_build.py
- pip3 install -e .
- if: tag IS present
stage: build
name: "Wallet Server Docker Image - Tagged Release"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
- docker push lbry/wallet-server:$TRAVIS_TAG
- if: tag IS blank AND branch = master
stage: build
name: "Wallet Server Docker Image - Master"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
- docker push lbry/wallet-server:master
notifications:
slack:
secure: "Am13HPtpgCMljh0MDVuoFHvQXB8yhf4Kvf/qAeSp5N0vsHGL70CSF9Ahccw8dVPE6mbuak1OGtSUb6/UaErLHkpz3ztaRLkDa9x7CmBB3Kynnh8oO2VbB7b/2ROULqkhF4VZmAnNfwrQrbC3gs8Sybp261Nyc7y4ww15xDYBrk2fyq4ds2DCaJdRxfJUJFonrZ6KXr3fVaXosO6cjuyS8eRodcmrqsT4cCtinjNTD1hGWoH107E4ObSmpVelxQO193KhNJMRiLlEcVkvYUOqIWBtwdGHbNE/6Yeuq1TXgKJ0KeJWAmW3wTfUYNngGXNAsyCnrhul5TKNevNzfIAQZHvRsczYiWPJV6LtohHT0CcUiCXJtvEPOyahEBfwK3etY/xxFqny7N9OEmpdW2sgsEPNPX2LJynJti2rQA9SuAD1ogR3ZpDy/NXoaAZf8PTdPcuNUMULV9PGG7tVrLBecO/W1qO6hdFxwlLdgqGLxAENZgGp++v/DhPk/WvtmHj7iTbRq0nxaTWyX5uKOn2ADH+k/yfutjv6BsQU9xNyPeZEEtuEpc6X6waiYn/8G9vl9PecvKC5H0MgsZ6asAxmg7mZ3VSMFG7mo8ENeOhSZ0Oz6ZTBILL3wFccZA9uJIq7NWmqC9dRiGiuKXBB62No7sINoHg3114e2xYa9qvNmGg="
cache:
directories:
- $HOME/.cache/pip
- $HOME/Library/Caches/pip
- $TRAVIS_BUILD_DIR/.tox
- $TRAVIS_BUILD_DIR/cache/wheel
addons:
#srcclr:
# debug: true
apt:
packages:
- libgmp3-dev
- build-essential
- git
- libssl-dev
- libffi-dev
before_install:
- virtualenv venv
- source venv/bin/activate
install:
- pip install -U pip
- pip install -r requirements.txt
- pip install .
script:
- pip install cython
- pip install mock pylint unqlite
- pylint lbrynet
- PYTHONPATH=. trial lbrynet.tests
- python -m unittest discover lbrynet/tests/integration -v
- rvm install ruby-2.3.1
- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
+44 -304
View File
@@ -4,324 +4,64 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/) with
regard to the json-rpc api. As we're currently pre-1.0 release, we
can and probably will change functionality and break backwards compatibility
can and probably will change functionality and break backwards compatability
at anytime.
## [0.30.1] - 2018-12-09
### API
* add `aioupnp_version` to `upnp` field in the response to `status`
* fixed uncaught claim signature errors
* `--blocking` support for `abandon` command (https://github.com/lbryio/lbry/pull/1618)
* don't create SSL keys if SSL is disabled (https://github.com/lbryio/lbry/pull/1644)
* added `account_send` command (https://github.com/lbryio/lbry/pull/1654)
* fixes for claim send to address (https://github.com/lbryio/lbry/pull/1651)
* better handle errors on resolve (https://github.com/lbryio/lbry/pull/1662)
* fix `transaction_show` (https://github.com/lbryio/lbry/pull/1689)
### Schema
* merged `lbryschema` repository into `lbrynet` (https://github.com/lbryio/lbry/pull/1444)
* updated `protobuf` to `3.2.0` (https://github.com/lbryio/lbry/issues/1602)
* use `types` repo for generating python protobufs (https://github.com/lbryio/lbry/pull/1605)
### Wallet
* fixed resolving names with invalid channel signatures
* fixed wallet sync issues
* fixed wallet locking/encryption issues
### DHT
* fixed retrying kademlia rpc calls (https://github.com/lbryio/lbry/pull/1495)
* fixed `KeyError` in `iterativeFindValue`, this is used by `peer_list` and when finding peers to download blobs from (https://github.com/lbryio/lbry/issues/1638)
* settings external ip address and ability to disable more DHT components (https://github.com/lbryio/lbry/pull/1604)
* improved routing table bucket initialization (https://github.com/lbryio/lbry/pull/1633)
### Database
* resolved too many variables in sqlite issue (https://github.com/lbryio/lbry/pull/1612)
### Misc
* switch to Python 3.7
* brought back icon for windows binary
* using custom list of mime types with more types (https://github.com/lbryio/lbry/pull/1700)
* add balance to startup report (https://github.com/lbryio/lbry/pull/1613)
* drop `six` dependency (https://github.com/lbryio/lbry/pull/1660)
* drop `mock` dependency (https://github.com/lbryio/lbry/pull/1661)
## [0.30.0] - 2018-10-22
This release is the most ambitious upgrade to lbrynet yet; including a brand
new wallet implementation, switch to Python 3 of the entire code base and
many changes to the API such as pagination and a more consistent output from
most commands.
## [Unreleased]
### Security
* upgraded `cryptography` package.
* switch to `coincurve` for private / public key generation (addresses)
* support for connecting to daemon via SSL
* improved workflow for encrypt/decrypt of wallet
*
*
### API
* unified all command line executables into a single `lbrynet` executable.
* added `use_https` configuration setting (off by default), if true the api
server will generate a self signed ssl certificate when started and only
handle api requests on the https address (default of `https://localhost:5279/lbryapi`)
* deprecated `daemon_stop` command, use `stop` instead.
* deprecated `wallet_balance` command, use `account_balance` instead.
* deprecated `wallet_unlock` command, use `account_unlock` instead.
* deprecated `wallet_decrypt` command, use `account_decrypt` instead.
* deprecated `wallet_encrypt` command, use `account_encrypt` instead.
* deprecated `wallet_prefill_addresses` command, use `account_fund` instead.
* deprecated `wallet_list` command, use `address_list` instead.
* deprecated `wallet_is_address_mine` command, use `address_is_mine` instead.
* deprecated `wallet_public_key` command.
* deprecated `wallet_new_address` command.
* deprecated `wallet_unused_address` command, use `address_unused` instead.
* deprecated `claim_renew` command.
* added `account_list` command to list accounts including their balance.
* added `account_add` command to add a previously created account from seed or private key.
* added `account_create` command to generate a new account.
* added `account_remove` command to remove an account from wallet.
* added `account_set` command to change a setting on an account.
* added `account_balance` command to get just the account balance.
* added `account_unlock` command to unlock an encrypted account.
* added `account_lock` command to lock an encrypted account.
* added `account_encrypt` command to encrypt an account.
* added `account_decrypt` command to decrypt an account.
* added `account_fund` command to move funds between or within an account in various ways.
* added `account_max_address_gap` command to find large gaps of unused addresses.
* added `address_list` command to list addresses.
* added `address_is_mine` command to check if an address is one of your addresses.
* added `address_unused` command to get existing or generate a new unused address.
* added pagination support for `address_list`, `channel_list`, `claim_list_mine`,
`transaction_list` and `utxo_list`.
* added `upnp` field to `status` response
* removed `send_amount_to_address` command previously marked as deprecated
* removed `channel_list_mine` command previously marked as deprecated
* removed `get_availability` command previously marked as deprecated
* updated `txupnp` dependency to `aioupnp` (UPnP for asyncio)
### Wallet
* changed to a new wallet implementation: [torba](https://github.com/lbryio/torba).
* changed wallet file format to support multiple accounts in one wallet.
* moved transaction data from wallet file into an sqlite database.
* changed channel certificates to be keyed by txid:nout instead of claim_id which
makes it possible to recover old certificates.
### File Manager
* Extensive internal changes as a result of porting to Python 3.
### DHT
* Extensive internal changes as a result of porting to Python 3.
### P2P
* Extensive internal changes as a result of porting to Python 3.
### Reflector
* No major changes to reflector.
### Database
* No changes to existing storage module.
### Known Issues
* Channel certificate import/export is currently not working.
## [0.21.2] - 2018-08-23
### Fixed
* issue in dht ping queue where enqueued pings that aren't yet due wouldn't be rescheduled
* blob mirror downloader not finishing streams that were partially uploaded at the time of the download attempt (https://github.com/lbryio/lbry/issues/1376)
## [0.21.1] - 2018-08-13
### Fixed
* `download_progress` field in `blockchain_headers` section of `status` not initializing correctly when resuming a download (https://github.com/lbryio/lbry/issues/1355)
* `wallet_send` not accepting decimal amounts (https://github.com/lbryio/lbry/issues/1356 commit https://github.com/lbryio/lbry/commit/1098ca0494ece420c70fc57f69d6d388715a99b8)
### Added
* `is_locked` to `wallet` in `status` response (https://github.com/lbryio/lbry/issues/1354, commit https://github.com/lbryio/lbry/commit/153022a1a7122ab6d31d3db433dccbe469bbcb3c)
### Changed
* Bumped `lbryum` requirement to 3.2.4 [see changelog](https://github.com/lbryio/lbryum/blob/master/CHANGELOG.md#324---2018-08-13)
## [0.21.0] - 2018-08-09
### Fixed
* check `claim_address` and `change_address` earlier on publishing, to avoid hard to understand errors later in the process (pr https://github.com/lbryio/lbry/pull/1347)
* loggly error reporting not following `share_usage_data` (pr https://github.com/lbryio/lbry/pull/1328)
* improper error handling when data is not valid JSON (pr https://github.com/lbryio/lbry/pull/1326)
* blob mirroring being set in unrelated tests, making them fail (pr https://github.com/lbryio/lbry/pull/1348)
* http blob mirroring edge cases (pr https://github.com/lbryio/lbry/pull/1315)
* external ports in log messages not showing the correct external port from the upnp redirects (https://github.com/lbryio/lbry/issues/1338) (pr https://github.com/lbryio/lbry/pull/1349)
* miniupnpc fallback issues in txupnp (https://github.com/lbryio/lbry/issues/1341) (pr https://github.com/lbryio/lbry/pull/1349)
* upnp error when disabled on router and a non-gateway is found, such as chromecast (https://github.com/lbryio/lbry/issues/1352) (https://github.com/lbryio/lbry/commit/dca4af942fbe95547794213775f0a62cd04a393f)
*
*
### Deprecated
* automatic claim renew, this is no longer needed
*
*
### Changed
* api server class to use components, and for all JSONRPC API commands to be callable so long as the required components are available. (pr https://github.com/lbryio/lbry/pull/1294)
* return error messages when required conditions on components are not met for API calls (pr https://github.com/lbryio/lbry/pull/1328)
* `status` to no longer return a base58 encoded `lbry_id`, instead return this as the hex encoded `node_id` in a new `dht` field. (pr https://github.com/lbryio/lbry/pull/1328)
* `startup_status` field in the response to `status` to be a dict of component names to status booleans (pr https://github.com/lbryio/lbry/pull/1328)
* renamed the `blockchain_status` field in the response to `status` to `wallet` (pr https://github.com/lbryio/lbry/pull/1328)
* moved and renamed `wallet_is_encrypted` to `is_encrypted` in the `wallet` field in the response to `status` (pr https://github.com/lbryio/lbry/pull/1328)
* moved wallet, upnp and dht startup code from `Session` to `Components` (pr https://github.com/lbryio/lbry/pull/1328)
* attempt blob downloads from http mirror sources (by default) concurrently to p2p sources (pr https://github.com/lbryio/lbry/pull/1233)
* replace miniupnpc with [txupnp](https://github.com/lbryio/txupnp). Since txupnp is still under development, it will internally fall back to miniupnpc. (pr https://github.com/lbryio/lbry/pull/1328)
* simplified test_misc.py in the functional tests (pr https://github.com/lbryio/lbry/pull/1328)
* update `cryptography` requirement to 2.3 (pr https://github.com/lbryio/lbry/pull/1333)
*
*
### Added
* `skipped_components` list to the response from `status` (pr https://github.com/lbryio/lbry/pull/1328)
* component statuses (`blockchain_headers`, `dht`, `wallet`, `blob_manager` `hash_announcer`, and `file_manager`) to the response to `status` (pr https://github.com/lbryio/lbry/pull/1328)
* `skipped_components` config setting, accepts a list of names of components to not run (pr https://github.com/lbryio/lbry/pull/1294)
* `ComponentManager` for managing the life-cycles of dependencies (pr https://github.com/lbryio/lbry/pull/1294)
* `requires` decorator to register the components required by a `jsonrpc_` command, to facilitate commands registering asynchronously (pr https://github.com/lbryio/lbry/pull/1294)
* unit tests for `ComponentManager` (pr https://github.com/lbryio/lbry/pull/1294)
* script to generate docs/api.json file (https://github.com/lbryio/lbry.tech/issues/42)
* additional information to the balance error message when editing a claim (pr https://github.com/lbryio/lbry/pull/1309)
* `address` and `port` arguments to `peer_ping` (https://github.com/lbryio/lbry/issues/1313) (pr https://github.com/lbryio/lbry/pull/1299)
* ability to download from HTTP mirrors by setting `download_mirrors` (prs https://github.com/lbryio/lbry/pull/1233 and https://github.com/lbryio/lbry/pull/1315)
* ability to filter peers from an iterative find value operation (finding peers for a blob). This is used to filter peers we've already found for a blob when accumulating the list of peers. (pr https://github.com/lbryio/lbry/pull/1287)
### Removed
* `session_status` argument and response field from `status` (pr https://github.com/lbryio/lbry/pull/1328)
* most of the internal attributes from `Daemon` (pr https://github.com/lbryio/lbry/pull/1294)
## [0.20.4] - 2018-07-18
### Fixed
* spelling errors in messages printed by `lbrynet-cli`
* high CPU usage when a stream is incomplete and the peers we're requesting from have no more blobs to send us (https://github.com/lbryio/lbry/pull/1301)
*
*
*
### Changed
* keep track of failures for DHT peers for up to ten minutes instead of indefinitely (https://github.com/lbryio/lbry/pull/1300)
* skip ignored peers from iterative lookups instead of blocking the peer who returned them to us too (https://github.com/lbryio/lbry/pull/1300)
* if a node becomes ignored during an iterative find cycle remove it from the shortlist so that we can't return it as a result nor try to probe it anyway (https://github.com/lbryio/lbry/pull/1303)
## [0.20.3] - 2018-07-03
### Fixed
* `blob_list` raising an error when blobs in a stream haven't yet been created (8a0d0b44ddf9cbeb2a9074eed39d6064ce21df64)
* stopping a download potentially raising an attribute error (https://github.com/lbryio/lbry/pull/1269)
* file manager startup locking up when there are many files for some channels (https://github.com/lbryio/lbry/pull/1281)
* improper sorting when getting the closest peers to a hash (https://github.com/lbryio/lbry/pull/1282)
### Changed
* raised the default `peer_search_timeout` setting from 3 to 30 and added logging for when it happens (https://github.com/lbryio/lbry/pull/1283)
* change iterative find stop condition on find value to allow it to continue until a value is found or it times out (https://github.com/lbryio/lbry/pull/1283)
* include all of our own blobs in the local dht datastore (as if we had announced them to ourselves) (https://github.com/lbryio/lbry/pull/1280)
* ignore dht `store` token validation errors for the first expiration-time after startup (fixes failed `store` requests after a restart) (https://github.com/lbryio/lbry/pull/1280)
### Removed
* `jsonrpclib` as a requirement for the project (https://github.com/lbryio/lbry/pull/1274)
## [0.20.2] - 2018-06-23
### Changed
* Bumped `lbryschema` requirement to 0.0.16 [see changelog](https://github.com/lbryio/lbryschema/blob/master/CHANGELOG.md#0016---2018-06-23)
* Bumped `lbryum` requirement to 3.2.3 [see changelog](https://github.com/lbryio/lbryum/blob/master/CHANGELOG.md#323---2018-06-23)
* save claims to sqlite in batches to speed up `resolve` queries for many uris
## [0.20.1] - 2018-06-19
### Fixed
* fixed token validation error when the dht node has just been started (https://github.com/lbryio/lbry/issues/1248)
* fixed a race condition when inserting a blob into the database (https://github.com/lbryio/lbry/issues/1129)
* reflector server incorrectly responding as if it has all the blobs for a stream that was only partially uploaded to it
* `publish` raising a database error when updating a claim that we don't have a file for (https://github.com/lbryio/lbry/issues/1165)
* blob client protocol not tearing itself down properly after a failure (https://github.com/lbryio/lbry/issues/950)
* lockup in wallet startup when one or more lbryumx servers are unavailable (https://github.com/lbryio/lbry/issues/1245)
* download being stopped if the sd blob downloaded and data did not start within the timeout (https://github.com/lbryio/lbry/issues/1172)
### Changed
* Bumped `lbryum` requirement to 3.2.2 [see changelog](https://github.com/lbryio/lbryum/blob/master/CHANGELOG.md#322---2018-06-19)
* `publish` to accept bid as a decimal string
## [0.20.0] - 2018-06-13
### TL;DR
This milestone release includes a large number of bug fixes, changes and additions covering all aspects of the daemon. Notable improvements include:
* Faster and more reliable downloading and uploading of content resulting from substantial work done on the Distributed Hash Table algorithms and protocol.
* Faster blockchain synchronization (headers) by downloading them from Amazon S3 under certain conditions.
* Faster overall app startup due to better optimized SQL queries.
* Power users of the `lbrynet-cli` will benefit from many bug fixes to commonly used commands and improvements in sorting of output.
* Please review the full change log for more details on specific fixes, changes and additions.
### Fixed
* fix payment rate manager typo ([1236](https://github.com/lbryio/lbry/pull/1236))
* handling error from dht clients with old `ping` method
* blobs not being re-announced if no peers successfully stored, now failed announcements are re-queued
* issue where an `AuthAPIClient` (used by `lbrynet-cli`) would fail to update its session secret and keep making new auth sessions, with every other request failing
* `use_auth_http` in a config file being overridden by the default command line argument to `lbrynet-daemon`, now the command line value will only override the config file value if it is provided
* `lbrynet-cli` not automatically switching to the authenticated client if the server is detected to be using authentication. This resulted in `lbrynet-cli` failing to run when `lbrynet-daemon` was run with the `--http-auth` flag
* fixed error when using `claim_show` with `txid` and `nout` arguments
* fixed error when saving server list to conf file ([1209](https://github.com/lbryio/lbry/pull/1209))
### Changed
* if the `use_authentication` setting is configured, use authentication for all api methods instead of only those with the `auth_required` decorator
* regenerate api keys on startup if the using authentication
* support both positional and keyword args for api calls
* `blob_announce` to queue the blob announcement but not block on it
* `peer_list` to return a list of dictionaries instead of a list of lists, added peer node ids to the results
* predictable result sorting for `claim_list` and `claim_list_mine` ([1216](https://github.com/lbryio/lbry/pull/1216) and [1208](https://github.com/lbryio/lbry/pull/1208))
* increase the default `auto_re_reflect_interval` setting to a day and the default `concurrent_announcers` setting to 10
* download blockchain headers from s3 before starting the wallet when the local height is more than `s3_headers_depth` (a config setting) blocks behind ([1177](https://github.com/lbryio/lbry/pull/1177))
* check headers file integrity on startup, removing/truncating the file to force re-download when necessary
* support partial headers file download from S3 ([1189](https://github.com/lbryio/lbry/pull/1189))
* refactor `add_completed_blobs` on storage.py, simplifying into less queries ([1226](https://github.com/lbryio/lbry/pull/1226))
* full verification of streams only during database migration instead of every startup ([1195](https://github.com/lbryio/lbry/pull/1195))
* database batching functions for starting up the file manager
* added `single_announce` and `last_announced_time` columns to the `blob` table in sqlite
* track successful reflector uploads in sqlite to minimize how many streams are attempted by auto re-reflect ([1194](https://github.com/lbryio/lbry/pull/1194))
* pass the sd hash to reflector ClientFactory instead of looking it up from the database
* dht logging to be more verbose with errors and warnings
* `store` kademlia rpc method to block on the call finishing and to return storing peer information
* kademlia protocol to minimally delay writes to the UDP socket
* several internal dht functions to use inlineCallbacks
* `DHTHashAnnouncer` and `Node` manage functions to use `LoopingCall`s instead of scheduling with `callLater`.
* refactored `DHTHashAnnouncer` to no longer use locks, use a `DeferredSemaphore` to limit concurrent announcers
* decoupled `DiskBlobManager` from `DHTHashAnnouncer`, get blob hashes to announce from `SQLiteStorage`. The blob manager no longer announces blobs after they are completed, the hash announcer takes care of this now.
* changed the bucket splitting condition in the dht routing table to be more aggressive
* ping dht nodes who have stored to us periodically to determine whether we should include them as an active peer for the hash when we are queried. Nodes that are known to be not reachable by the node storing the record are no longer returned as peers by the storing node.
* changed dht bootstrap join process to better populate the routing table initially
* cache dht node tokens used during announcement to minimize the number of requests that are needed
* implement BEP0005 dht rules to classify nodes as good, bad, or unknown and for when to add them to the routing table (http://www.bittorrent.org/beps/bep_0005.html)
* refactored internal dht contact class to track failure counts/times, the time the contact last replied to us, and the time the node last requested something fom us ([1211](https://github.com/lbryio/lbry/pull/1211))
* refactored dht iterativeFind
* sort dht contacts returned by `findCloseNodes` in the routing table
* `reactor` and `callLater`, `listenUDP`, and `resolve` functions to be configurable (to allow easier testing)
* calls to get the current time to use `reactor.seconds` (to control callLater and LoopingCall timing in tests)
* temporarily disabled data price negotiation, treat all data as free
* disabled Cryptonator price feed
* use `treq` instead of `txrequests` ([1191](https://github.com/lbryio/lbry/pull/1191))
* updated `cryptography` version to 2.2.2
* removed `pycrypto` dependency, replacing all calls to `cryptography`
* several internal dht functions to use inlineCallbacks
* `DHTHashAnnouncer` and `Node` manage functions to use `LoopingCall`s instead of scheduling with `callLater`.
* `store` kademlia rpc method to block on the call finishing and to return storing peer information
* refactored `DHTHashAnnouncer` to longer use locks, use a `DeferredSemaphore` to limit concurrent announcers
* decoupled `DiskBlobManager` from `DHTHashAnnouncer`
* blob hashes to announce to be controlled by`SQLiteStorage`
* kademlia protocol to not delay writes to the UDP socket
* `reactor` and `callLater`, `listenUDP`, and `resolve` functions to be configurable (to allow easier testing)
* calls to get the current time to use `reactor.seconds` (to control callLater and LoopingCall timing in tests)
* `blob_announce` to queue the blob announcement but not block on it
* blob completion to not `callLater` an immediate announce, let `SQLiteStorage` and the `DHTHashAnnouncer` handle it
* raise the default number of concurrent blob announcers to 100
* dht logging to be more verbose with errors and warnings
* added `single_announce` and `last_announced_time` columns to the `blob` table in sqlite
* pass the sd hash to reflector ClientFactory instead of looking it up
* if the `use_authentication` setting is configured, use authentication for all api methods instead of only those with the `auth_required` decorator
* regenerate api keys on startup if the using authentication
* support both positional and keyword args for api calls
* `peer_list` to return a list of dictionaries instead of a list of lists, added peer node ids to the results
* download blockchain headers from s3 before starting the wallet when the local height is more than `s3_headers_depth` (a config setting) blocks behind
### Added
* `peer_ping` command
* `--sort` option in `file_list` ([1174](https://github.com/lbryio/lbry/pull/1174))
* `port` field to contacts returned by `routing_table_get`
* configurable `concurrent_announcers` and `s3_headers_depth` settings
* virtual kademlia network and mock udp transport for dht integration tests
* functional tests for bootstrapping the dht, announcing and expiring hashes, finding and pinging nodes, protocol version 0/1 backwards/forwards compatibility, and rejoining the network
* linux distro and desktop name added to analytics ([1218](https://github.com/lbryio/lbry/pull/1218))
* certifi module for Twisted SSL verification on Windows ([1213](https://github.com/lbryio/lbry/pull/1213))
* protocol version to dht requests and to the response from `findValue`
* virtual kademlia network and mock udp transport for dht integration tests
* integration tests for bootstrapping the dht
* configurable `concurrent_announcers` and `s3_headers_depth` settings
* `peer_ping` command
### Removed
* `announce_all` argument from `blob_announce`
* old `blob_announce_all` command
* unused `--wallet` argument to `lbrynet-daemon`, which used to be to support `PTCWallet`.
* `AuthJSONRPCServer.auth_required` decorator ([1161](https://github.com/lbryio/lbry/pull/1161))
* `OptimizedTreeRoutingTable` class used by the dht node for the time being
*
*
## [0.19.3] - 2018-05-04
### Changed
* download blockchain headers from s3 before starting the wallet when the local height is more than s3_headers_depth (a config setting) blocks behind (https://github.com/lbryio/lbry/pull/1177)
* un-deprecated report_bug command (https://github.com/lbryio/lbry/commit/f8e418fb4448a3ed1531657f8b3c608fb568af85)
## [0.19.2] - 2018-03-28
### Fixed
* incorrectly raised download cancelled error for already verified blob files
@@ -486,7 +226,7 @@ most commands.
* Fixed https://github.com/lbryio/lbry/issues/923
* Fixed concurrent reflects opening too many files
* Fixed cases when reflecting would fail on error conditions
* Fixed deadlocks from occurring during blob writes
* Fixed deadlocks from occuring during blob writes
* Fixed and updated`lbrynet.tests.dht`
* Fixed redundant dht id
* Fixed dht `ping` method
@@ -873,7 +613,7 @@ most commands.
* Removed check_pending logic from Daemon
* Switched to txrequests so requests can use twisted event loop
* Renamed API command file_seed to file_set_status
* Don't add expected payment to wallet when payment rate is 0
* Dont add expected payment to wallet when payment rate is 0
### Fixed
* Fix restart procedure in DaemonControl
* Create download directory if it doesn't exist
@@ -904,7 +644,7 @@ most commands.
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
* Remove unused API commands from daemon
* Fix file filter `outpoint`
* Made dictionary key names in API command outputs to be more consistent
* Made dictionary key names in API commmand outputs to be more consistent
### Added
* Add file filters: `claim_id`, `outpoint`, and `rowid`
* Make loggly logs less verbose
@@ -956,7 +696,7 @@ most commands.
* Include download url in version check
### Fixed
* add missing traceback to logging
* add misssing traceback to logging
## [0.8.3] - 2017-02-15
### Fixed
+65 -36
View File
@@ -1,48 +1,70 @@
# Installing LBRY
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
The recommended way to install LBRY is to use an installer. We provide installers for all major operating systems. See the [README](README.md).
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
Here's a video walkthrough of this setup, which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch):
[![Setup for development](https://spee.ch/2018-10-04-17-13-54-017046806.png)](https://spee.ch/967f99344308f1e90f0620d91b6c93e4dfb240e0/lbrynet-dev-setup.mp4)
These instructions are for installing from source code, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
## Prerequisites
Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred). Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
Before starting, you'll need to install some additional libraries depending on your operating system.
After installing python 3, you'll need to install some additional libraries depending on your operating system.
### Python
### Virtualenv
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
### Windows
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
LBRY requires Python 2.7. Python 3 is not supported.
### macOS
### OS X Prerequisites
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
Install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
Remaining dependencies can then be installed by running:
```
brew install python3 protobuf
brew install mpfr libmpc protobuf
sudo easy_install pip
sudo pip install virtualenv
```
### Linux
### Linux Prerequisites
On Ubuntu (we recommend 18.04), install the following:
On Ubuntu (we recommend 16.04), install the following:
```
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
sudo apt-get install build-essential python2.7 python2.7-dev python-pip git python-virtualenv libssl-dev libffi-dev python-protobuf
```
On Raspbian, you will also need to install `python-pyparsing`.
If you're running another Linux distro, install the equivalent of the above packages for your system.
If you're running another Linux flavor, install the equivalent of the above packages for your system.
### Windows Prerequisites
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
Add C:\MinGW\bin to the windows PATH.
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing:
```
[build]
compiler = mingw32
```
If using virtualenv, which is recommended, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
Currently, available binaries include:
- Cython
- Twisted
- Zope.interface
- pywin32
- requests
Install the above binaries with `pip install *.whl`
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
Finally, you'll need [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9). Download the source and compile with MinGW by running `mingw32make.bat`. Then install the module by running `python setupmingw32.py install`.
## Installation
@@ -51,27 +73,34 @@ To install:
```
git clone https://github.com/lbryio/lbry.git
cd lbry
virtualenv lbry-venv --python=python3.7
virtualenv lbry-venv
source lbry-venv/bin/activate
python --version # Python 2 is not supported. Make sure you're on Python 3.7
pip install -e .
python --version # Python 3 is not supported. Make sure you're on Python 2.7
pip install -U -r requirements.txt
pip install --editable .
```
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
To start LBRY, run `lbrynet-daemon` in a terminal.
## Run the tests
To run the unit tests from the repo directory:
```
trial --reactor=asyncio tests.unit
```
To verify your install, `which lbrynet-daemon` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
## Usage
## Making a Change
To start the API server:
`lbrynet start`
If you are not already inside the virtual environment, run:
```
source lbry-venv/bin/activate
```
Then run:
```
pip install --editable .
```
This will install `lbrynet-daemon` in such a way that the changes you make to the code will be used as soon as you restart the daemon.
Happy hacking!
+34 -38
View File
@@ -1,56 +1,52 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
[![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry)
[![Coverage Status](https://coveralls.io/repos/github/lbryio/lbry/badge.svg)](https://coveralls.io/github/lbryio/lbry)
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
# LBRY
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized network.
* Built on Python 3.7+ and `asyncio`.
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
This repo is a reference implementation of the [LBRY API](https://lbry.io/api).
## Installation
It provides a daemon that can interact with the network via a json-rpc interface over HTTP.
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
## Installing
## Usage
**Note**: This project no longer bundles a graphic interface (browser). If you want to use LBRY via a GUI, [use the LBRY App](https://github.com/lbryio/lbry-app).
Run `lbrynet start` to launch the API server.
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases, for macOS, Debian-based Linux, and Windows.
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
Installing from source is also relatively painless, full instructions are in [INSTALL.md](INSTALL.md)
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
## Running
With the daemon running, `lbrynet commands` will show you a list of commands.
Run `lbrynet-daemon` to launch the daemon.
The full API is documented [here](https://lbry.tech/api/sdk).
## Using
## Running from source
By default, `lbrynet-daemon` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
Our [quickstart guide](http://lbry.io/quickstart) provides a simple walkthrough and examples for learning.
## Contributing
The full API is documented [here](https://lbryio.github.io/lbry/cli).
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.io/faq/contributing) link.
## What is LBRY?
LBRY is a protocol that provides a fully decentralized network for the discovery, distribution, and payment of data.
It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, and rights and access rules.
LBRY also provides a data network consists of peers uploading and downloading data from other peers, possibly in exchange for payments, and a distributed hash table, used by peers to discover other peers.
## Contributions
To contribute, [join us on Discord](https://chat.lbry.io/) or contact jeremy@lbry.io. Pull requests are also welcome.
## Support
Please open an issue and describe your situation in detail. We will respond as soon as we can.
For private issues, contact jeremy@lbry.io.
## License
This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
## Contact
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
## Additional information and links
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry/blob/master/example_daemon_settings.yml).
See [LICENSE](LICENSE)
-5
View File
@@ -1,5 +0,0 @@
build:
python setup.py apk
adb install -r lbry-0.1-service-debug.apk
.PHONY: build
-49
View File
@@ -1,49 +0,0 @@
import inspect
from os.path import dirname, join
import sh
from pythonforandroid.util import ensure_dir
from pythonforandroid.toolchain import (
Bootstrap, current_directory, info, info_main, shprint
)
class LBRYServiceBootstrap(Bootstrap):
name = 'lbry-service'
recipe_depends = ['genericndkbuild', 'python3']
bootstrap_dir = dirname(__file__)
def get_common_dir(self):
return join(dirname(inspect.getfile(Bootstrap)), 'bootstraps', 'common')
def run_distribute(self):
info_main('# Creating Android project from build and {} bootstrap'.format(
self.name))
info('This currently just copies the build stuff straight from the build dir.')
shprint(sh.rm, '-rf', self.dist_dir)
shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
with current_directory(self.dist_dir):
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
arch = self.ctx.archs[0]
if len(self.ctx.archs) > 1:
raise ValueError('built for more than one arch, but bootstrap cannot handle that yet')
info('Bootstrap running with arch {}'.format(arch))
with current_directory(self.dist_dir):
info('Copying python distribution')
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
self.distribute_aars(arch)
self.distribute_javaclasses(self.ctx.javaclass_dir)
python_bundle_dir = join('_python_bundle', '_python_bundle')
ensure_dir(python_bundle_dir)
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
join(self.dist_dir, python_bundle_dir), arch)
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super().run_distribute()
-89
View File
@@ -1,89 +0,0 @@
# prevent user to include invalid extensions
*.apk
*.pxd
# eggs
*.egg-info
# unit test
unittest/*
# python config
config/makesetup
# unused kivy files (platform specific)
kivy/input/providers/wm_*
kivy/input/providers/mactouch*
kivy/input/providers/probesysfs*
kivy/input/providers/mtdev*
kivy/input/providers/hidinput*
kivy/core/camera/camera_videocapture*
kivy/core/spelling/*osx*
kivy/core/video/video_pyglet*
kivy/tools
kivy/tests/*
kivy/*/*.h
kivy/*/*.pxi
# unused encodings
lib-dynload/*codec*
encodings/cp*.pyo
encodings/tis*
encodings/shift*
encodings/bz2*
encodings/iso*
encodings/undefined*
encodings/johab*
encodings/p*
encodings/m*
encodings/euc*
encodings/k*
encodings/unicode_internal*
encodings/quo*
encodings/gb*
encodings/big5*
encodings/hp*
encodings/hz*
# unused python modules
bsddb/*
wsgiref/*
hotshot/*
pydoc_data/*
tty.pyo
anydbm.pyo
nturl2path.pyo
LICENCE.txt
macurl2path.pyo
dummy_threading.pyo
audiodev.pyo
antigravity.pyo
dumbdbm.pyo
sndhdr.pyo
__phello__.foo.pyo
sunaudio.pyo
os2emxpath.pyo
multiprocessing/dummy*
# unused binaries python modules
lib-dynload/termios.so
lib-dynload/_lsprof.so
lib-dynload/*audioop.so
lib-dynload/_hotshot.so
lib-dynload/_heapq.so
lib-dynload/_json.so
lib-dynload/grp.so
lib-dynload/resource.so
lib-dynload/pyexpat.so
lib-dynload/_ctypes_test.so
lib-dynload/_testcapi.so
# odd files
plat-linux3/regen
#>sqlite3
# conditionnal include depending if some recipes are included or not.
sqlite3/*
lib-dynload/_sqlite3.so
#<sqlite3
-1
View File
@@ -1 +0,0 @@
include $(call all-subdir-makefiles)
@@ -1,7 +0,0 @@
# Uncomment this if you're using STL in your project
# See CPLUSPLUS-SUPPORT.html in the NDK documentation for more information
# APP_STL := stlport_static
# APP_ABI := armeabi armeabi-v7a x86
APP_ABI := $(ARCH)
@@ -1,22 +0,0 @@
LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE := main
# Add your application source files here...
LOCAL_SRC_FILES := start.c pyjniusjni.c
LOCAL_CFLAGS += -I$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_INCLUDE_ROOT) $(EXTRA_CFLAGS)
LOCAL_SHARED_LIBRARIES := python_shared
LOCAL_LDLIBS := -llog $(EXTRA_LDLIBS)
LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../../other_builds/$(MK_PYTHON_LINK_ROOT) $(APPLICATION_ADDITIONAL_LDFLAGS)
include $(BUILD_SHARED_LIBRARY)
ifdef CRYSTAX_PYTHON_VERSION
$(call import-module,python/$(CRYSTAX_PYTHON_VERSION))
endif
@@ -1,10 +0,0 @@
LOCAL_PATH := $(call my-dir)
include $(CLEAR_VARS)
LOCAL_MODULE := main
LOCAL_SRC_FILES := YourSourceHere.c
include $(BUILD_SHARED_LIBRARY)
$(call import-module,SDL)LOCAL_PATH := $(call my-dir)
@@ -1,6 +0,0 @@
#define BOOTSTRAP_NAME_SERVICEONLY
#define BOOTSTRAP_USES_NO_SDL_HEADERS
const char bootstrap_name[] = "service_only";
@@ -1,103 +0,0 @@
#include <pthread.h>
#include <jni.h>
#define LOGI(...) do {} while (0)
#define LOGE(...) do {} while (0)
#include "android/log.h"
/* These JNI management functions are taken from SDL2, but modified to refer to pyjnius */
/* #define LOG(n, x) __android_log_write(ANDROID_LOG_INFO, (n), (x)) */
/* #define LOGP(x) LOG("python", (x)) */
#define LOG_TAG "Python_android"
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
/* Function headers */
JNIEnv* Android_JNI_GetEnv(void);
static void Android_JNI_ThreadDestroyed(void*);
static pthread_key_t mThreadKey;
static JavaVM* mJavaVM;
int Android_JNI_SetupThread(void)
{
Android_JNI_GetEnv();
return 1;
}
/* Library init */
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
{
JNIEnv *env;
mJavaVM = vm;
LOGI("JNI_OnLoad called");
if ((*mJavaVM)->GetEnv(mJavaVM, (void**) &env, JNI_VERSION_1_4) != JNI_OK) {
LOGE("Failed to get the environment using GetEnv()");
return -1;
}
/*
* Create mThreadKey so we can keep track of the JNIEnv assigned to each thread
* Refer to http://developer.android.com/guide/practices/design/jni.html for the rationale behind this
*/
if (pthread_key_create(&mThreadKey, Android_JNI_ThreadDestroyed) != 0) {
__android_log_print(ANDROID_LOG_ERROR, "pyjniusjni", "Error initializing pthread key");
}
Android_JNI_SetupThread();
return JNI_VERSION_1_4;
}
JNIEnv* Android_JNI_GetEnv(void)
{
/* From http://developer.android.com/guide/practices/jni.html
* All threads are Linux threads, scheduled by the kernel.
* They're usually started from managed code (using Thread.start), but they can also be created elsewhere and then
* attached to the JavaVM. For example, a thread started with pthread_create can be attached with the
* JNI AttachCurrentThread or AttachCurrentThreadAsDaemon functions. Until a thread is attached, it has no JNIEnv,
* and cannot make JNI calls.
* Attaching a natively-created thread causes a java.lang.Thread object to be constructed and added to the "main"
* ThreadGroup, making it visible to the debugger. Calling AttachCurrentThread on an already-attached thread
* is a no-op.
* Note: You can call this function any number of times for the same thread, there's no harm in it
*/
JNIEnv *env;
int status = (*mJavaVM)->AttachCurrentThread(mJavaVM, &env, NULL);
if(status < 0) {
LOGE("failed to attach current thread");
return 0;
}
/* From http://developer.android.com/guide/practices/jni.html
* Threads attached through JNI must call DetachCurrentThread before they exit. If coding this directly is awkward,
* in Android 2.0 (Eclair) and higher you can use pthread_key_create to define a destructor function that will be
* called before the thread exits, and call DetachCurrentThread from there. (Use that key with pthread_setspecific
* to store the JNIEnv in thread-local-storage; that way it'll be passed into your destructor as the argument.)
* Note: The destructor is not called unless the stored value is != NULL
* Note: You can call this function any number of times for the same thread, there's no harm in it
* (except for some lost CPU cycles)
*/
pthread_setspecific(mThreadKey, (void*) env);
return env;
}
static void Android_JNI_ThreadDestroyed(void* value)
{
/* The thread is being destroyed, detach it from the Java VM and set the mThreadKey value to NULL as required */
JNIEnv *env = (JNIEnv*) value;
if (env != NULL) {
(*mJavaVM)->DetachCurrentThread(mJavaVM);
pthread_setspecific(mThreadKey, NULL);
}
}
void *WebView_AndroidGetJNIEnv()
{
return Android_JNI_GetEnv();
}
Whitespace-only changes.
@@ -1,141 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
/**
* @author Kamran Zafar
*
*/
public class Octal {
/**
* Parse an octal string from a header buffer. This is used for the file
* permission mode value.
*
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The long value of the octal string.
*/
public static long parseOctal(byte[] header, int offset, int length) {
long result = 0;
boolean stillPadding = true;
int end = offset + length;
for (int i = offset; i < end; ++i) {
if (header[i] == 0)
break;
if (header[i] == (byte) ' ' || header[i] == '0') {
if (stillPadding)
continue;
if (header[i] == (byte) ' ')
break;
}
stillPadding = false;
result = ( result << 3 ) + ( header[i] - '0' );
}
return result;
}
/**
* Parse an octal integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The integer value of the octal bytes.
*/
public static int getOctalBytes(long value, byte[] buf, int offset, int length) {
int idx = length - 1;
buf[offset + idx] = 0;
--idx;
buf[offset + idx] = (byte) ' ';
--idx;
if (value == 0) {
buf[offset + idx] = (byte) '0';
--idx;
} else {
for (long val = value; idx >= 0 && val > 0; --idx) {
buf[offset + idx] = (byte) ( (byte) '0' + (byte) ( val & 7 ) );
val = val >> 3;
}
}
for (; idx >= 0; --idx) {
buf[offset + idx] = (byte) ' ';
}
return offset + length;
}
/**
* Parse the checksum octal integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The integer value of the entry's checksum.
*/
public static int getCheckSumOctalBytes(long value, byte[] buf, int offset, int length) {
getOctalBytes( value, buf, offset, length );
buf[offset + length - 1] = (byte) ' ';
buf[offset + length - 2] = 0;
return offset + length;
}
/**
* Parse an octal long integer from a header buffer.
*
* @param value
* @param buf
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
*
* @return The long value of the octal bytes.
*/
public static int getLongOctalBytes(long value, byte[] buf, int offset, int length) {
byte[] temp = new byte[length + 1];
getOctalBytes( value, temp, 0, length + 1 );
System.arraycopy( temp, 0, buf, offset, length );
return offset + length;
}
}
@@ -1,28 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
/**
* @author Kamran Zafar
*
*/
public class TarConstants {
public static final int EOF_BLOCK = 1024;
public static final int DATA_BLOCK = 512;
public static final int HEADER_BLOCK = 512;
}
@@ -1,284 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
import java.util.Date;
/**
* @author Kamran Zafar
*
*/
public class TarEntry {
protected File file;
protected TarHeader header;
private TarEntry() {
this.file = null;
header = new TarHeader();
}
public TarEntry(File file, String entryName) {
this();
this.file = file;
this.extractTarHeader(entryName);
}
public TarEntry(byte[] headerBuf) {
this();
this.parseTarHeader(headerBuf);
}
/**
* Constructor to create an entry from an existing TarHeader object.
*
* This method is useful to add new entries programmatically (e.g. for
* adding files or directories that do not exist in the file system).
*
* @param header
*
*/
public TarEntry(TarHeader header) {
this.file = null;
this.header = header;
}
public boolean equals(TarEntry it) {
return header.name.toString().equals(it.header.name.toString());
}
public boolean isDescendent(TarEntry desc) {
return desc.header.name.toString().startsWith(header.name.toString());
}
public TarHeader getHeader() {
return header;
}
public String getName() {
String name = header.name.toString();
if (header.namePrefix != null && !header.namePrefix.toString().equals("")) {
name = header.namePrefix.toString() + "/" + name;
}
return name;
}
public void setName(String name) {
header.name = new StringBuffer(name);
}
public int getUserId() {
return header.userId;
}
public void setUserId(int userId) {
header.userId = userId;
}
public int getGroupId() {
return header.groupId;
}
public void setGroupId(int groupId) {
header.groupId = groupId;
}
public String getUserName() {
return header.userName.toString();
}
public void setUserName(String userName) {
header.userName = new StringBuffer(userName);
}
public String getGroupName() {
return header.groupName.toString();
}
public void setGroupName(String groupName) {
header.groupName = new StringBuffer(groupName);
}
public void setIds(int userId, int groupId) {
this.setUserId(userId);
this.setGroupId(groupId);
}
public void setModTime(long time) {
header.modTime = time / 1000;
}
public void setModTime(Date time) {
header.modTime = time.getTime() / 1000;
}
public Date getModTime() {
return new Date(header.modTime * 1000);
}
public File getFile() {
return this.file;
}
public long getSize() {
return header.size;
}
public void setSize(long size) {
header.size = size;
}
/**
* Checks if the org.kamrazafar.jtar entry is a directory
*
* @return
*/
public boolean isDirectory() {
if (this.file != null)
return this.file.isDirectory();
if (header != null) {
if (header.linkFlag == TarHeader.LF_DIR)
return true;
if (header.name.toString().endsWith("/"))
return true;
}
return false;
}
/**
* Extract header from File
*
* @param entryName
*/
public void extractTarHeader(String entryName) {
header = TarHeader.createHeader(entryName, file.length(), file.lastModified() / 1000, file.isDirectory());
}
/**
* Calculate checksum
*
* @param buf
* @return
*/
public long computeCheckSum(byte[] buf) {
long sum = 0;
for (int i = 0; i < buf.length; ++i) {
sum += 255 & buf[i];
}
return sum;
}
/**
* Writes the header to the byte buffer
*
* @param outbuf
*/
public void writeEntryHeader(byte[] outbuf) {
int offset = 0;
offset = TarHeader.getNameBytes(header.name, outbuf, offset, TarHeader.NAMELEN);
offset = Octal.getOctalBytes(header.mode, outbuf, offset, TarHeader.MODELEN);
offset = Octal.getOctalBytes(header.userId, outbuf, offset, TarHeader.UIDLEN);
offset = Octal.getOctalBytes(header.groupId, outbuf, offset, TarHeader.GIDLEN);
long size = header.size;
offset = Octal.getLongOctalBytes(size, outbuf, offset, TarHeader.SIZELEN);
offset = Octal.getLongOctalBytes(header.modTime, outbuf, offset, TarHeader.MODTIMELEN);
int csOffset = offset;
for (int c = 0; c < TarHeader.CHKSUMLEN; ++c)
outbuf[offset++] = (byte) ' ';
outbuf[offset++] = header.linkFlag;
offset = TarHeader.getNameBytes(header.linkName, outbuf, offset, TarHeader.NAMELEN);
offset = TarHeader.getNameBytes(header.magic, outbuf, offset, TarHeader.USTAR_MAGICLEN);
offset = TarHeader.getNameBytes(header.userName, outbuf, offset, TarHeader.USTAR_USER_NAMELEN);
offset = TarHeader.getNameBytes(header.groupName, outbuf, offset, TarHeader.USTAR_GROUP_NAMELEN);
offset = Octal.getOctalBytes(header.devMajor, outbuf, offset, TarHeader.USTAR_DEVLEN);
offset = Octal.getOctalBytes(header.devMinor, outbuf, offset, TarHeader.USTAR_DEVLEN);
offset = TarHeader.getNameBytes(header.namePrefix, outbuf, offset, TarHeader.USTAR_FILENAME_PREFIX);
for (; offset < outbuf.length;)
outbuf[offset++] = 0;
long checkSum = this.computeCheckSum(outbuf);
Octal.getCheckSumOctalBytes(checkSum, outbuf, csOffset, TarHeader.CHKSUMLEN);
}
/**
* Parses the tar header to the byte buffer
*
* @param header
* @param bh
*/
public void parseTarHeader(byte[] bh) {
int offset = 0;
header.name = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
offset += TarHeader.NAMELEN;
header.mode = (int) Octal.parseOctal(bh, offset, TarHeader.MODELEN);
offset += TarHeader.MODELEN;
header.userId = (int) Octal.parseOctal(bh, offset, TarHeader.UIDLEN);
offset += TarHeader.UIDLEN;
header.groupId = (int) Octal.parseOctal(bh, offset, TarHeader.GIDLEN);
offset += TarHeader.GIDLEN;
header.size = Octal.parseOctal(bh, offset, TarHeader.SIZELEN);
offset += TarHeader.SIZELEN;
header.modTime = Octal.parseOctal(bh, offset, TarHeader.MODTIMELEN);
offset += TarHeader.MODTIMELEN;
header.checkSum = (int) Octal.parseOctal(bh, offset, TarHeader.CHKSUMLEN);
offset += TarHeader.CHKSUMLEN;
header.linkFlag = bh[offset++];
header.linkName = TarHeader.parseName(bh, offset, TarHeader.NAMELEN);
offset += TarHeader.NAMELEN;
header.magic = TarHeader.parseName(bh, offset, TarHeader.USTAR_MAGICLEN);
offset += TarHeader.USTAR_MAGICLEN;
header.userName = TarHeader.parseName(bh, offset, TarHeader.USTAR_USER_NAMELEN);
offset += TarHeader.USTAR_USER_NAMELEN;
header.groupName = TarHeader.parseName(bh, offset, TarHeader.USTAR_GROUP_NAMELEN);
offset += TarHeader.USTAR_GROUP_NAMELEN;
header.devMajor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
offset += TarHeader.USTAR_DEVLEN;
header.devMinor = (int) Octal.parseOctal(bh, offset, TarHeader.USTAR_DEVLEN);
offset += TarHeader.USTAR_DEVLEN;
header.namePrefix = TarHeader.parseName(bh, offset, TarHeader.USTAR_FILENAME_PREFIX);
}
}
@@ -1,243 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
/**
* Header
*
* <pre>
* Offset Size Field
* 0 100 File name
* 100 8 File mode
* 108 8 Owner's numeric user ID
* 116 8 Group's numeric user ID
* 124 12 File size in bytes
* 136 12 Last modification time in numeric Unix time format
* 148 8 Checksum for header block
* 156 1 Link indicator (file type)
* 157 100 Name of linked file
* </pre>
*
*
* File Types
*
* <pre>
* Value Meaning
* '0' Normal file
* (ASCII NUL) Normal file (now obsolete)
* '1' Hard link
* '2' Symbolic link
* '3' Character special
* '4' Block special
* '5' Directory
* '6' FIFO
* '7' Contigous
* </pre>
*
*
*
* Ustar header
*
* <pre>
* Offset Size Field
* 257 6 UStar indicator "ustar"
* 263 2 UStar version "00"
* 265 32 Owner user name
* 297 32 Owner group name
* 329 8 Device major number
* 337 8 Device minor number
* 345 155 Filename prefix
* </pre>
*/
public class TarHeader {
/*
* Header
*/
public static final int NAMELEN = 100;
public static final int MODELEN = 8;
public static final int UIDLEN = 8;
public static final int GIDLEN = 8;
public static final int SIZELEN = 12;
public static final int MODTIMELEN = 12;
public static final int CHKSUMLEN = 8;
public static final byte LF_OLDNORM = 0;
/*
* File Types
*/
public static final byte LF_NORMAL = (byte) '0';
public static final byte LF_LINK = (byte) '1';
public static final byte LF_SYMLINK = (byte) '2';
public static final byte LF_CHR = (byte) '3';
public static final byte LF_BLK = (byte) '4';
public static final byte LF_DIR = (byte) '5';
public static final byte LF_FIFO = (byte) '6';
public static final byte LF_CONTIG = (byte) '7';
/*
* Ustar header
*/
public static final String USTAR_MAGIC = "ustar"; // POSIX
public static final int USTAR_MAGICLEN = 8;
public static final int USTAR_USER_NAMELEN = 32;
public static final int USTAR_GROUP_NAMELEN = 32;
public static final int USTAR_DEVLEN = 8;
public static final int USTAR_FILENAME_PREFIX = 155;
// Header values
public StringBuffer name;
public int mode;
public int userId;
public int groupId;
public long size;
public long modTime;
public int checkSum;
public byte linkFlag;
public StringBuffer linkName;
public StringBuffer magic; // ustar indicator and version
public StringBuffer userName;
public StringBuffer groupName;
public int devMajor;
public int devMinor;
public StringBuffer namePrefix;
public TarHeader() {
this.magic = new StringBuffer(TarHeader.USTAR_MAGIC);
this.name = new StringBuffer();
this.linkName = new StringBuffer();
String user = System.getProperty("user.name", "");
if (user.length() > 31)
user = user.substring(0, 31);
this.userId = 0;
this.groupId = 0;
this.userName = new StringBuffer(user);
this.groupName = new StringBuffer("");
this.namePrefix = new StringBuffer();
}
/**
* Parse an entry name from a header buffer.
*
* @param name
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The header's entry name.
*/
public static StringBuffer parseName(byte[] header, int offset, int length) {
StringBuffer result = new StringBuffer(length);
int end = offset + length;
for (int i = offset; i < end; ++i) {
if (header[i] == 0)
break;
result.append((char) header[i]);
}
return result;
}
/**
* Determine the number of bytes in an entry name.
*
* @param name
* @param header
* The header buffer from which to parse.
* @param offset
* The offset into the buffer from which to parse.
* @param length
* The number of header bytes to parse.
* @return The number of bytes in a header's entry name.
*/
public static int getNameBytes(StringBuffer name, byte[] buf, int offset, int length) {
int i;
for (i = 0; i < length && i < name.length(); ++i) {
buf[offset + i] = (byte) name.charAt(i);
}
for (; i < length; ++i) {
buf[offset + i] = 0;
}
return offset + length;
}
/**
* Creates a new header for a file/directory entry.
*
*
* @param name
* File name
* @param size
* File size in bytes
* @param modTime
* Last modification time in numeric Unix time format
* @param dir
* Is directory
*
* @return
*/
public static TarHeader createHeader(String entryName, long size, long modTime, boolean dir) {
String name = entryName;
name = TarUtils.trim(name.replace(File.separatorChar, '/'), '/');
TarHeader header = new TarHeader();
header.linkName = new StringBuffer("");
if (name.length() > 100) {
header.namePrefix = new StringBuffer(name.substring(0, name.lastIndexOf('/')));
header.name = new StringBuffer(name.substring(name.lastIndexOf('/') + 1));
} else {
header.name = new StringBuffer(name);
}
if (dir) {
header.mode = 040755;
header.linkFlag = TarHeader.LF_DIR;
if (header.name.charAt(header.name.length() - 1) != '/') {
header.name.append("/");
}
header.size = 0;
} else {
header.mode = 0100644;
header.linkFlag = TarHeader.LF_NORMAL;
header.size = size;
}
header.modTime = modTime;
header.checkSum = 0;
header.devMajor = 0;
header.devMinor = 0;
return header;
}
}
@@ -1,249 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
/**
* @author Kamran Zafar
*
*/
public class TarInputStream extends FilterInputStream {
private static final int SKIP_BUFFER_SIZE = 2048;
private TarEntry currentEntry;
private long currentFileSize;
private long bytesRead;
private boolean defaultSkip = false;
public TarInputStream(InputStream in) {
super(in);
currentFileSize = 0;
bytesRead = 0;
}
@Override
public boolean markSupported() {
return false;
}
/**
* Not supported
*
*/
@Override
public synchronized void mark(int readlimit) {
}
/**
* Not supported
*
*/
@Override
public synchronized void reset() throws IOException {
throw new IOException("mark/reset not supported");
}
/**
* Read a byte
*
* @see java.io.FilterInputStream#read()
*/
@Override
public int read() throws IOException {
byte[] buf = new byte[1];
int res = this.read(buf, 0, 1);
if (res != -1) {
return 0xFF & buf[0];
}
return res;
}
/**
* Checks if the bytes being read exceed the entry size and adjusts the byte
* array length. Updates the byte counters
*
*
* @see java.io.FilterInputStream#read(byte[], int, int)
*/
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (currentEntry != null) {
if (currentFileSize == currentEntry.getSize()) {
return -1;
} else if ((currentEntry.getSize() - currentFileSize) < len) {
len = (int) (currentEntry.getSize() - currentFileSize);
}
}
int br = super.read(b, off, len);
if (br != -1) {
if (currentEntry != null) {
currentFileSize += br;
}
bytesRead += br;
}
return br;
}
/**
* Returns the next entry in the tar file
*
* @return TarEntry
* @throws IOException
*/
public TarEntry getNextEntry() throws IOException {
closeCurrentEntry();
byte[] header = new byte[TarConstants.HEADER_BLOCK];
byte[] theader = new byte[TarConstants.HEADER_BLOCK];
int tr = 0;
// Read full header
while (tr < TarConstants.HEADER_BLOCK) {
int res = read(theader, 0, TarConstants.HEADER_BLOCK - tr);
if (res < 0) {
break;
}
System.arraycopy(theader, 0, header, tr, res);
tr += res;
}
// Check if record is null
boolean eof = true;
for (byte b : header) {
if (b != 0) {
eof = false;
break;
}
}
if (!eof) {
currentEntry = new TarEntry(header);
}
return currentEntry;
}
/**
* Returns the current offset (in bytes) from the beginning of the stream.
* This can be used to find out at which point in a tar file an entry's content begins, for instance.
*/
public long getCurrentOffset() {
return bytesRead;
}
/**
* Closes the current tar entry
*
* @throws IOException
*/
protected void closeCurrentEntry() throws IOException {
if (currentEntry != null) {
if (currentEntry.getSize() > currentFileSize) {
// Not fully read, skip rest of the bytes
long bs = 0;
while (bs < currentEntry.getSize() - currentFileSize) {
long res = skip(currentEntry.getSize() - currentFileSize - bs);
if (res == 0 && currentEntry.getSize() - currentFileSize > 0) {
// I suspect file corruption
throw new IOException("Possible tar file corruption");
}
bs += res;
}
}
currentEntry = null;
currentFileSize = 0L;
skipPad();
}
}
/**
* Skips the pad at the end of each tar entry file content
*
* @throws IOException
*/
protected void skipPad() throws IOException {
if (bytesRead > 0) {
int extra = (int) (bytesRead % TarConstants.DATA_BLOCK);
if (extra > 0) {
long bs = 0;
while (bs < TarConstants.DATA_BLOCK - extra) {
long res = skip(TarConstants.DATA_BLOCK - extra - bs);
bs += res;
}
}
}
}
/**
* Skips 'n' bytes on the InputStream<br>
* Overrides default implementation of skip
*
*/
@Override
public long skip(long n) throws IOException {
if (defaultSkip) {
// use skip method of parent stream
// may not work if skip not implemented by parent
long bs = super.skip(n);
bytesRead += bs;
return bs;
}
if (n <= 0) {
return 0;
}
long left = n;
byte[] sBuff = new byte[SKIP_BUFFER_SIZE];
while (left > 0) {
int res = read(sBuff, 0, (int) (left < SKIP_BUFFER_SIZE ? left : SKIP_BUFFER_SIZE));
if (res < 0) {
break;
}
left -= res;
}
return n - left;
}
public boolean isDefaultSkip() {
return defaultSkip;
}
public void setDefaultSkip(boolean defaultSkip) {
this.defaultSkip = defaultSkip;
}
}
@@ -1,163 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
/**
* @author Kamran Zafar
*
*/
public class TarOutputStream extends OutputStream {
private final OutputStream out;
private long bytesWritten;
private long currentFileSize;
private TarEntry currentEntry;
public TarOutputStream(OutputStream out) {
this.out = out;
bytesWritten = 0;
currentFileSize = 0;
}
public TarOutputStream(final File fout) throws FileNotFoundException {
this.out = new BufferedOutputStream(new FileOutputStream(fout));
bytesWritten = 0;
currentFileSize = 0;
}
/**
* Opens a file for writing.
*/
public TarOutputStream(final File fout, final boolean append) throws IOException {
@SuppressWarnings("resource")
RandomAccessFile raf = new RandomAccessFile(fout, "rw");
final long fileSize = fout.length();
if (append && fileSize > TarConstants.EOF_BLOCK) {
raf.seek(fileSize - TarConstants.EOF_BLOCK);
}
out = new BufferedOutputStream(new FileOutputStream(raf.getFD()));
}
/**
* Appends the EOF record and closes the stream
*
* @see java.io.FilterOutputStream#close()
*/
@Override
public void close() throws IOException {
closeCurrentEntry();
write( new byte[TarConstants.EOF_BLOCK] );
out.close();
}
/**
* Writes a byte to the stream and updates byte counters
*
* @see java.io.FilterOutputStream#write(int)
*/
@Override
public void write(int b) throws IOException {
out.write( b );
bytesWritten += 1;
if (currentEntry != null) {
currentFileSize += 1;
}
}
/**
* Checks if the bytes being written exceed the current entry size.
*
* @see java.io.FilterOutputStream#write(byte[], int, int)
*/
@Override
public void write(byte[] b, int off, int len) throws IOException {
if (currentEntry != null && !currentEntry.isDirectory()) {
if (currentEntry.getSize() < currentFileSize + len) {
throw new IOException( "The current entry[" + currentEntry.getName() + "] size["
+ currentEntry.getSize() + "] is smaller than the bytes[" + ( currentFileSize + len )
+ "] being written." );
}
}
out.write( b, off, len );
bytesWritten += len;
if (currentEntry != null) {
currentFileSize += len;
}
}
/**
* Writes the next tar entry header on the stream
*
* @param entry
* @throws IOException
*/
public void putNextEntry(TarEntry entry) throws IOException {
closeCurrentEntry();
byte[] header = new byte[TarConstants.HEADER_BLOCK];
entry.writeEntryHeader( header );
write( header );
currentEntry = entry;
}
/**
* Closes the current tar entry
*
* @throws IOException
*/
protected void closeCurrentEntry() throws IOException {
if (currentEntry != null) {
if (currentEntry.getSize() > currentFileSize) {
throw new IOException( "The current entry[" + currentEntry.getName() + "] of size["
+ currentEntry.getSize() + "] has not been fully written." );
}
currentEntry = null;
currentFileSize = 0;
pad();
}
}
/**
* Pads the last content block
*
* @throws IOException
*/
protected void pad() throws IOException {
if (bytesWritten > 0) {
int extra = (int) ( bytesWritten % TarConstants.DATA_BLOCK );
if (extra > 0) {
write( new byte[TarConstants.DATA_BLOCK - extra] );
}
}
}
}
@@ -1,96 +0,0 @@
/**
* Copyright 2012 Kamran Zafar
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.kamranzafar.jtar;
import java.io.File;
/**
* @author Kamran
*
*/
public class TarUtils {
/**
* Determines the tar file size of the given folder/file path
*
* @param path
* @return
*/
public static long calculateTarSize(File path) {
return tarSize(path) + TarConstants.EOF_BLOCK;
}
private static long tarSize(File dir) {
long size = 0;
if (dir.isFile()) {
return entrySize(dir.length());
} else {
File[] subFiles = dir.listFiles();
if (subFiles != null && subFiles.length > 0) {
for (File file : subFiles) {
if (file.isFile()) {
size += entrySize(file.length());
} else {
size += tarSize(file);
}
}
} else {
// Empty folder header
return TarConstants.HEADER_BLOCK;
}
}
return size;
}
private static long entrySize(long fileSize) {
long size = 0;
size += TarConstants.HEADER_BLOCK; // Header
size += fileSize; // File size
long extra = size % TarConstants.DATA_BLOCK;
if (extra > 0) {
size += (TarConstants.DATA_BLOCK - extra); // pad
}
return size;
}
public static String trim(String s, char c) {
StringBuffer tmp = new StringBuffer(s);
for (int i = 0; i < tmp.length(); i++) {
if (tmp.charAt(i) != c) {
break;
} else {
tmp.deleteCharAt(i);
}
}
for (int i = tmp.length() - 1; i >= 0; i--) {
if (tmp.charAt(i) != c) {
break;
} else {
tmp.deleteCharAt(i);
}
}
return tmp.toString();
}
}
@@ -1,400 +0,0 @@
package org.kivy.android;
import java.net.Socket;
import java.net.InetSocketAddress;
import android.os.SystemClock;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.ArrayList;
import android.app.*;
import android.content.*;
import android.view.*;
import android.view.SurfaceView;
import android.app.Activity;
import android.content.Intent;
import android.util.Log;
import android.widget.Toast;
import android.os.AsyncTask;
import android.os.Bundle;
import android.os.PowerManager;
import android.graphics.PixelFormat;
import android.view.SurfaceHolder;
import android.content.Context;
import android.content.pm.ActivityInfo;
import android.content.pm.PackageManager;
import android.content.pm.ApplicationInfo;
import android.content.Intent;
import android.widget.ImageView;
import java.io.InputStream;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Color;
import android.widget.AbsoluteLayout;
import android.webkit.WebViewClient;
import android.webkit.WebView;
import org.kivy.android.PythonUtil;
import org.renpy.android.ResourceManager;
import org.renpy.android.AssetExtract;
public class PythonActivity extends Activity {
// This activity is modified from a mixture of the SDLActivity and
// PythonActivity in the SDL2 bootstrap, but removing all the SDL2
// specifics.
private static final String TAG = "PythonActivity";
public static PythonActivity mActivity = null;
/** If shared libraries (e.g. the native application) could not be loaded. */
public static boolean mBrokenLibraries;
protected static Thread mPythonThread;
private ResourceManager resourceManager = null;
private Bundle mMetaData = null;
private PowerManager.WakeLock mWakeLock = null;
public String getAppRoot() {
String app_root = getFilesDir().getAbsolutePath() + "/app";
return app_root;
}
public static void initialize() {
// The static nature of the singleton and Android quirkiness force us to initialize everything here
// Otherwise, when exiting the app and returning to it, these variables *keep* their pre exit values
mBrokenLibraries = false;
}
@Override
protected void onCreate(Bundle savedInstanceState) {
Log.v(TAG, "My oncreate running");
resourceManager = new ResourceManager(this);
Log.v(TAG, "Ready to unpack");
File app_root_file = new File(getAppRoot());
unpackData("private", app_root_file);
Log.v(TAG, "About to do super onCreate");
super.onCreate(savedInstanceState);
Log.v(TAG, "Did super onCreate");
this.mActivity = this;
//this.showLoadingScreen();
Log.v("Python", "Device: " + android.os.Build.DEVICE);
Log.v("Python", "Model: " + android.os.Build.MODEL);
//Log.v(TAG, "Ready to unpack");
//new UnpackFilesTask().execute(getAppRoot());
PythonActivity.initialize();
// Load shared libraries
String errorMsgBrokenLib = "";
try {
loadLibraries();
} catch(UnsatisfiedLinkError e) {
System.err.println(e.getMessage());
mBrokenLibraries = true;
errorMsgBrokenLib = e.getMessage();
} catch(Exception e) {
System.err.println(e.getMessage());
mBrokenLibraries = true;
errorMsgBrokenLib = e.getMessage();
}
if (mBrokenLibraries)
{
AlertDialog.Builder dlgAlert = new AlertDialog.Builder(this);
dlgAlert.setMessage("An error occurred while trying to load the application libraries. Please try again and/or reinstall."
+ System.getProperty("line.separator")
+ System.getProperty("line.separator")
+ "Error: " + errorMsgBrokenLib);
dlgAlert.setTitle("Python Error");
dlgAlert.setPositiveButton("Exit",
new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialog,int id) {
// if this button is clicked, close current activity
PythonActivity.mActivity.finish();
}
});
dlgAlert.setCancelable(false);
dlgAlert.create().show();
return;
}
// Set up the Python environment
String app_root_dir = getAppRoot();
String mFilesDirectory = mActivity.getFilesDir().getAbsolutePath();
Log.v(TAG, "Setting env vars for start.c and Python to use");
PythonActivity.nativeSetEnv("ANDROID_ENTRYPOINT", "main.pyo");
PythonActivity.nativeSetEnv("ANDROID_ARGUMENT", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_APP_PATH", app_root_dir);
PythonActivity.nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
PythonActivity.nativeSetEnv("ANDROID_UNPACK", app_root_dir);
PythonActivity.nativeSetEnv("PYTHONHOME", app_root_dir);
PythonActivity.nativeSetEnv("PYTHONPATH", app_root_dir + ":" + app_root_dir + "/lib");
PythonActivity.nativeSetEnv("PYTHONOPTIMIZE", "2");
try {
Log.v(TAG, "Access to our meta-data...");
mActivity.mMetaData = mActivity.getPackageManager().getApplicationInfo(
mActivity.getPackageName(), PackageManager.GET_META_DATA).metaData;
PowerManager pm = (PowerManager) mActivity.getSystemService(Context.POWER_SERVICE);
if ( mActivity.mMetaData.getInt("wakelock") == 1 ) {
mActivity.mWakeLock = pm.newWakeLock(PowerManager.SCREEN_BRIGHT_WAKE_LOCK, "Screen On");
mActivity.mWakeLock.acquire();
}
} catch (PackageManager.NameNotFoundException e) {
}
final Thread pythonThread = new Thread(new PythonMain(), "PythonThread");
PythonActivity.mPythonThread = pythonThread;
pythonThread.start();
}
@Override
public void onDestroy() {
Log.i("Destroy", "end of app");
super.onDestroy();
// make sure all child threads (python_thread) are stopped
android.os.Process.killProcess(android.os.Process.myPid());
}
public void loadLibraries() {
String app_root = new String(getAppRoot());
File app_root_file = new File(app_root);
PythonUtil.loadLibraries(app_root_file);
}
public void recursiveDelete(File f) {
if (f.isDirectory()) {
for (File r : f.listFiles()) {
recursiveDelete(r);
}
}
f.delete();
}
/**
* Show an error using a toast. (Only makes sense from non-UI
* threads.)
*/
public void toastError(final String msg) {
final Activity thisActivity = this;
runOnUiThread(new Runnable () {
public void run() {
Toast.makeText(thisActivity, msg, Toast.LENGTH_LONG).show();
}
});
// Wait to show the error.
synchronized (this) {
try {
this.wait(1000);
} catch (InterruptedException e) {
}
}
}
public void unpackData(final String resource, File target) {
Log.v(TAG, "UNPACKING!!! " + resource + " " + target.getName());
// The version of data in memory and on disk.
String data_version = resourceManager.getString(resource + "_version");
String disk_version = null;
Log.v(TAG, "Data version is " + data_version);
// If no version, no unpacking is necessary.
if (data_version == null) {
return;
}
// Check the current disk version, if any.
String filesDir = target.getAbsolutePath();
String disk_version_fn = filesDir + "/" + resource + ".version";
try {
byte buf[] = new byte[64];
InputStream is = new FileInputStream(disk_version_fn);
int len = is.read(buf);
disk_version = new String(buf, 0, len);
is.close();
} catch (Exception e) {
disk_version = "";
}
// If the disk data is out of date, extract it and write the
// version file.
// if (! data_version.equals(disk_version)) {
if (! data_version.equals(disk_version)) {
Log.v(TAG, "Extracting " + resource + " assets.");
recursiveDelete(target);
target.mkdirs();
AssetExtract ae = new AssetExtract(this);
if (!ae.extractTar(resource + ".mp3", target.getAbsolutePath())) {
toastError("Could not extract " + resource + " data.");
}
try {
// Write .nomedia.
new File(target, ".nomedia").createNewFile();
// Write version file.
FileOutputStream os = new FileOutputStream(disk_version_fn);
os.write(data_version.getBytes());
os.close();
} catch (Exception e) {
Log.w("python", e);
}
}
}
long lastBackClick = SystemClock.elapsedRealtime();
@Override
public boolean onKeyDown(int keyCode, KeyEvent event) {
// If it wasn't the Back key or there's no web page history, bubble up to the default
// system behavior (probably exit the activity)
if (SystemClock.elapsedRealtime() - lastBackClick > 2000){
lastBackClick = SystemClock.elapsedRealtime();
Toast.makeText(this, "Click again to close the app",
Toast.LENGTH_LONG).show();
return true;
}
lastBackClick = SystemClock.elapsedRealtime();
return super.onKeyDown(keyCode, event);
}
//----------------------------------------------------------------------------
// Listener interface for onNewIntent
//
public interface NewIntentListener {
void onNewIntent(Intent intent);
}
private List<NewIntentListener> newIntentListeners = null;
public void registerNewIntentListener(NewIntentListener listener) {
if ( this.newIntentListeners == null )
this.newIntentListeners = Collections.synchronizedList(new ArrayList<NewIntentListener>());
this.newIntentListeners.add(listener);
}
public void unregisterNewIntentListener(NewIntentListener listener) {
if ( this.newIntentListeners == null )
return;
this.newIntentListeners.remove(listener);
}
@Override
protected void onNewIntent(Intent intent) {
if ( this.newIntentListeners == null )
return;
this.onResume();
synchronized ( this.newIntentListeners ) {
Iterator<NewIntentListener> iterator = this.newIntentListeners.iterator();
while ( iterator.hasNext() ) {
(iterator.next()).onNewIntent(intent);
}
}
}
//----------------------------------------------------------------------------
// Listener interface for onActivityResult
//
public interface ActivityResultListener {
void onActivityResult(int requestCode, int resultCode, Intent data);
}
private List<ActivityResultListener> activityResultListeners = null;
public void registerActivityResultListener(ActivityResultListener listener) {
if ( this.activityResultListeners == null )
this.activityResultListeners = Collections.synchronizedList(new ArrayList<ActivityResultListener>());
this.activityResultListeners.add(listener);
}
public void unregisterActivityResultListener(ActivityResultListener listener) {
if ( this.activityResultListeners == null )
return;
this.activityResultListeners.remove(listener);
}
@Override
protected void onActivityResult(int requestCode, int resultCode, Intent intent) {
if ( this.activityResultListeners == null )
return;
this.onResume();
synchronized ( this.activityResultListeners ) {
Iterator<ActivityResultListener> iterator = this.activityResultListeners.iterator();
while ( iterator.hasNext() )
(iterator.next()).onActivityResult(requestCode, resultCode, intent);
}
}
public static void start_service(String serviceTitle, String serviceDescription,
String pythonServiceArgument) {
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
String argument = PythonActivity.mActivity.getFilesDir().getAbsolutePath();
String filesDirectory = argument;
String app_root_dir = PythonActivity.mActivity.getAppRoot();
serviceIntent.putExtra("androidPrivate", argument);
serviceIntent.putExtra("androidArgument", app_root_dir);
serviceIntent.putExtra("serviceEntrypoint", "service/main.pyo");
serviceIntent.putExtra("pythonName", "python");
serviceIntent.putExtra("pythonHome", app_root_dir);
serviceIntent.putExtra("pythonPath", app_root_dir + ":" + app_root_dir + "/lib");
serviceIntent.putExtra("serviceTitle", serviceTitle);
serviceIntent.putExtra("serviceDescription", serviceDescription);
serviceIntent.putExtra("pythonServiceArgument", pythonServiceArgument);
PythonActivity.mActivity.startService(serviceIntent);
}
public static void stop_service() {
Intent serviceIntent = new Intent(PythonActivity.mActivity, PythonService.class);
PythonActivity.mActivity.stopService(serviceIntent);
}
public static native void nativeSetEnv(String j_name, String j_value);
public static native int nativeInit(Object arguments);
}
class PythonMain implements Runnable {
@Override
public void run() {
PythonActivity.nativeInit(new String[0]);
}
}
@@ -1,45 +0,0 @@
package org.kivy.android.concurrency;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* Created by ryan on 3/28/14.
*/
public class PythonEvent {
private final Lock lock = new ReentrantLock();
private final Condition cond = lock.newCondition();
private boolean flag = false;
public void set() {
lock.lock();
try {
flag = true;
cond.signalAll();
} finally {
lock.unlock();
}
}
public void wait_() throws InterruptedException {
lock.lock();
try {
while (!flag) {
cond.await();
}
} finally {
lock.unlock();
}
}
public void clear() {
lock.lock();
try {
flag = false;
cond.signalAll();
} finally {
lock.unlock();
}
}
}
@@ -1,19 +0,0 @@
package org.kivy.android.concurrency;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* Created by ryan on 3/28/14.
*/
public class PythonLock {
private final Lock lock = new ReentrantLock();
public void acquire() {
lock.lock();
}
public void release() {
lock.unlock();
}
}
@@ -1,257 +0,0 @@
package org.renpy.android;
import java.util.List;
import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.Intent;
import android.content.IntentFilter;
import android.hardware.Sensor;
import android.hardware.SensorEvent;
import android.hardware.SensorEventListener;
import android.hardware.SensorManager;
import android.net.ConnectivityManager;
import android.net.NetworkInfo;
import android.net.wifi.ScanResult;
import android.net.wifi.WifiManager;
import android.os.Vibrator;
import android.view.View;
/**
* Methods that are expected to be called via JNI, to access the device's
* non-screen hardware. (For example, the vibration and accelerometer.)
*/
public class Hardware {
// The context.
static Context context;
static View view;
/**
* Vibrate for s seconds.
*/
public static void vibrate(double s) {
Vibrator v = (Vibrator) context
.getSystemService(Context.VIBRATOR_SERVICE);
if (v != null) {
v.vibrate((int) (1000 * s));
}
}
/**
* Get an Overview of all Hardware Sensors of an Android Device
*/
public static String getHardwareSensors() {
SensorManager sm = (SensorManager) context
.getSystemService(Context.SENSOR_SERVICE);
List<Sensor> allSensors = sm.getSensorList(Sensor.TYPE_ALL);
if (allSensors != null) {
String resultString = "";
for (Sensor s : allSensors) {
resultString += String.format("Name=" + s.getName());
resultString += String.format(",Vendor=" + s.getVendor());
resultString += String.format(",Version=" + s.getVersion());
resultString += String.format(",MaximumRange="
+ s.getMaximumRange());
// XXX MinDelay is not in the 2.2
// resultString += String.format(",MinDelay=" +
// s.getMinDelay());
resultString += String.format(",Power=" + s.getPower());
resultString += String.format(",Type=" + s.getType() + "\n");
}
return resultString;
}
return "";
}
/**
* Get Access to 3 Axis Hardware Sensors Accelerometer, Orientation and
* Magnetic Field Sensors
*/
public static class generic3AxisSensor implements SensorEventListener {
private final SensorManager sSensorManager;
private final Sensor sSensor;
private final int sSensorType;
SensorEvent sSensorEvent;
public generic3AxisSensor(int sensorType) {
sSensorType = sensorType;
sSensorManager = (SensorManager) context
.getSystemService(Context.SENSOR_SERVICE);
sSensor = sSensorManager.getDefaultSensor(sSensorType);
}
public void onAccuracyChanged(Sensor sensor, int accuracy) {
}
public void onSensorChanged(SensorEvent event) {
sSensorEvent = event;
}
/**
* Enable or disable the Sensor by registering/unregistering
*/
public void changeStatus(boolean enable) {
if (enable) {
sSensorManager.registerListener(this, sSensor,
SensorManager.SENSOR_DELAY_NORMAL);
} else {
sSensorManager.unregisterListener(this, sSensor);
}
}
/**
* Read the Sensor
*/
public float[] readSensor() {
if (sSensorEvent != null) {
return sSensorEvent.values;
} else {
float rv[] = { 0f, 0f, 0f };
return rv;
}
}
}
public static generic3AxisSensor accelerometerSensor = null;
public static generic3AxisSensor orientationSensor = null;
public static generic3AxisSensor magneticFieldSensor = null;
/**
* functions for backward compatibility reasons
*/
public static void accelerometerEnable(boolean enable) {
if (accelerometerSensor == null)
accelerometerSensor = new generic3AxisSensor(
Sensor.TYPE_ACCELEROMETER);
accelerometerSensor.changeStatus(enable);
}
public static float[] accelerometerReading() {
float rv[] = { 0f, 0f, 0f };
if (accelerometerSensor == null)
return rv;
return (float[]) accelerometerSensor.readSensor();
}
public static void orientationSensorEnable(boolean enable) {
if (orientationSensor == null)
orientationSensor = new generic3AxisSensor(Sensor.TYPE_ORIENTATION);
orientationSensor.changeStatus(enable);
}
public static float[] orientationSensorReading() {
float rv[] = { 0f, 0f, 0f };
if (orientationSensor == null)
return rv;
return (float[]) orientationSensor.readSensor();
}
public static void magneticFieldSensorEnable(boolean enable) {
if (magneticFieldSensor == null)
magneticFieldSensor = new generic3AxisSensor(
Sensor.TYPE_MAGNETIC_FIELD);
magneticFieldSensor.changeStatus(enable);
}
public static float[] magneticFieldSensorReading() {
float rv[] = { 0f, 0f, 0f };
if (magneticFieldSensor == null)
return rv;
return (float[]) magneticFieldSensor.readSensor();
}
/**
* Scan WiFi networks
*/
static List<ScanResult> latestResult;
public static void enableWifiScanner() {
IntentFilter i = new IntentFilter();
i.addAction(WifiManager.SCAN_RESULTS_AVAILABLE_ACTION);
context.registerReceiver(new BroadcastReceiver() {
@Override
public void onReceive(Context c, Intent i) {
// Code to execute when SCAN_RESULTS_AVAILABLE_ACTION event
// occurs
WifiManager w = (WifiManager) c
.getSystemService(Context.WIFI_SERVICE);
latestResult = w.getScanResults(); // Returns a <list> of
// scanResults
}
}, i);
}
public static String scanWifi() {
// Now you can call this and it should execute the broadcastReceiver's
// onReceive()
WifiManager wm = (WifiManager) context
.getSystemService(Context.WIFI_SERVICE);
boolean a = wm.startScan();
if (latestResult != null) {
String latestResultString = "";
for (ScanResult result : latestResult) {
latestResultString += String.format("%s\t%s\t%d\n",
result.SSID, result.BSSID, result.level);
}
return latestResultString;
}
return "";
}
/**
* network state
*/
public static boolean network_state = false;
/**
* Check network state directly
*
* (only one connection can be active at a given moment, detects all network
* type)
*
*/
public static boolean checkNetwork() {
boolean state = false;
final ConnectivityManager conMgr = (ConnectivityManager) context
.getSystemService(Context.CONNECTIVITY_SERVICE);
final NetworkInfo activeNetwork = conMgr.getActiveNetworkInfo();
if (activeNetwork != null && activeNetwork.isConnected()) {
state = true;
} else {
state = false;
}
return state;
}
/**
* To recieve network state changes
*/
public static void registerNetworkCheck() {
IntentFilter i = new IntentFilter();
i.addAction(ConnectivityManager.CONNECTIVITY_ACTION);
context.registerReceiver(new BroadcastReceiver() {
@Override
public void onReceive(Context c, Intent i) {
network_state = checkNetwork();
}
}, i);
}
}
Whitespace-only changes.
Whitespace-only changes.
@@ -1,93 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="{{ args.package }}"
android:versionCode="{{ args.numeric_version }}"
android:versionName="{{ args.version }}"
android:installLocation="auto">
<supports-screens
android:smallScreens="true"
android:normalScreens="true"
android:largeScreens="true"
android:anyDensity="true"
{% if args.min_sdk_version >= 9 %}
android:xlargeScreens="true"
{% endif %}
/>
<!-- Android 2.3.3 -->
<uses-sdk android:minSdkVersion="{{ args.min_sdk_version }}" android:targetSdkVersion="{{ android_api }}" />
<!-- Set permissions -->
{% for perm in args.permissions %}
{% if '.' in perm %}
<uses-permission android:name="{{ perm }}" />
{% else %}
<uses-permission android:name="android.permission.{{ perm }}" />
{% endif %}
{% endfor %}
{% if args.wakelock %}
<uses-permission android:name="android.permission.WAKE_LOCK" />
{% endif %}
{% if args.billing_pubkey %}
<uses-permission android:name="com.android.vending.BILLING" />
{% endif %}
<!-- Create a Java class extending SDLActivity and place it in a
directory under src matching the package, e.g.
src/com/gamemaker/game/MyGame.java
then replace "SDLActivity" with the name of your class (e.g. "MyGame")
in the XML below.
An example Java class can be found in README-android.txt
-->
<application android:label="@string/app_name"
android:icon="@drawable/icon"
android:allowBackup="true"
android:theme="@android:style/Theme.NoTitleBar{% if not args.window %}.Fullscreen{% endif %}"
android:hardwareAccelerated="true" >
{% for m in args.meta_data %}
<meta-data android:name="{{ m.split('=', 1)[0] }}" android:value="{{ m.split('=', 1)[-1] }}"/>{% endfor %}
<meta-data android:name="wakelock" android:value="{% if args.wakelock %}1{% else %}0{% endif %}"/>
<activity android:name="org.kivy.android.PythonActivity"
android:label="@string/app_name"
android:configChanges="keyboardHidden|orientation{% if args.min_sdk_version >= 13 %}|screenSize{% endif %}"
>
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
{%- if args.intent_filters -%}
{{- args.intent_filters -}}
{%- endif -%}
</activity>
{% if service %}
<service android:name="org.kivy.android.PythonService"
android:process=":pythonservice" />
{% endif %}
{% for name in service_names %}
<service android:name="{{ args.package }}.Service{{ name|capitalize }}"
android:process=":service_{{ name }}" />
{% endfor %}
{% if args.billing_pubkey %}
<service android:name="org.kivy.android.billing.BillingReceiver"
android:process=":pythonbilling" />
<receiver android:name="org.kivy.android.billing.BillingReceiver"
android:process=":pythonbillingreceiver">
<intent-filter>
<action android:name="com.android.vending.billing.IN_APP_NOTIFY" />
<action android:name="com.android.vending.billing.RESPONSE_CODE" />
<action android:name="com.android.vending.billing.PURCHASE_STATE_CHANGED" />
</intent-filter>
</receiver>
{% endif %}
</application>
</manifest>
@@ -1,74 +0,0 @@
package {{ args.package }};
import android.os.Binder;
import android.os.IBinder;
import android.content.Intent;
import android.content.Context;
import org.kivy.android.PythonService;
public class Service{{ name|capitalize }} extends PythonService {
/**
* Binder given to clients
*/
private final IBinder mBinder = new Service{{ name|capitalize }}Binder();
{% if sticky %}
/**
* {@inheritDoc}
*/
@Override
public int getStartType() {
return START_STICKY;
}
{% endif %}
{% if foreground %}
/**
* {@inheritDoc}
*/
@Override
public boolean getStartForeground() {
return true;
}
{% endif %}
public static void start(Context ctx, String pythonServiceArgument) {
String argument = ctx.getFilesDir().getAbsolutePath() + "/app";
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
intent.putExtra("androidPrivate", argument);
intent.putExtra("androidArgument", argument);
intent.putExtra("serviceEntrypoint", "{{ entrypoint }}");
intent.putExtra("serviceTitle", "{{ name|capitalize }}");
intent.putExtra("serviceDescription", "");
intent.putExtra("pythonName", "{{ name }}");
intent.putExtra("pythonHome", argument);
intent.putExtra("androidUnpack", argument);
intent.putExtra("pythonPath", argument + ":" + argument + "/lib");
intent.putExtra("pythonServiceArgument", pythonServiceArgument);
ctx.startService(intent);
}
public static void stop(Context ctx) {
Intent intent = new Intent(ctx, Service{{ name|capitalize }}.class);
ctx.stopService(intent);
}
/**
* {@inheritDoc}
*/
@Override
public IBinder onBind(Intent intent) {
return mBinder;
}
/**
* Class used for the client Binder. Because we know this service always
* runs in the same process as its clients, we don't need to deal with IPC.
*/
public class Service{{ name|capitalize }}Binder extends Binder {
Service{{ name|capitalize }} getService() {
// Return this instance of Service{{ name|capitalize }} so clients can call public methods
return Service{{ name|capitalize }}.this;
}
}
}
@@ -1,5 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>
<string name="app_name">{{ args.name }}</string>
<string name="private_version">{{ private_version }}</string>
</resources>
-68
View File
@@ -1,68 +0,0 @@
from lbrynet.extras.daemon.Daemon import Daemon
from jnius import PythonJavaClass, autoclass, java_method
JAVA_NAMESPACE = 'org.kivy.android'
JNI_NAMESPACE = 'org/kivy/android'
_activity = autoclass(JAVA_NAMESPACE + '.PythonActivity').mActivity
_callbacks = {
'on_new_intent': [],
'on_activity_result': [],
}
class NewIntentListener(PythonJavaClass):
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$NewIntentListener']
__javacontext__ = 'app'
def __init__(self, callback, **kwargs):
super(NewIntentListener, self).__init__(**kwargs)
self.callback = callback
@java_method('(Landroid/content/Intent;)V')
def onNewIntent(self, intent):
self.callback(intent)
class ActivityResultListener(PythonJavaClass):
__javainterfaces__ = [JNI_NAMESPACE + '/PythonActivity$ActivityResultListener']
__javacontext__ = 'app'
def __init__(self, callback):
super(ActivityResultListener, self).__init__()
self.callback = callback
@java_method('(IILandroid/content/Intent;)V')
def onActivityResult(self, requestCode, resultCode, intent):
self.callback(requestCode, resultCode, intent)
def bind(**kwargs):
for event, callback in kwargs.items():
if event not in _callbacks:
raise Exception('Unknown {!r} event'.format(event))
elif event == 'on_new_intent':
listener = NewIntentListener(callback)
_activity.registerNewIntentListener(listener)
_callbacks[event].append(listener)
elif event == 'on_activity_result':
listener = ActivityResultListener(callback)
_activity.registerActivityResultListener(listener)
_callbacks[event].append(listener)
def unbind(**kwargs):
for event, callback in kwargs.items():
if event not in _callbacks:
raise Exception('Unknown {!r} event'.format(event))
else:
for listener in _callbacks[event][:]:
if listener.callback == callback:
_callbacks[event].remove(listener)
if event == 'on_new_intent':
_activity.unregisterNewIntentListener(listener)
elif event == 'on_activity_result':
_activity.unregisterActivityResultListener(listener)
print('YEAH! Done all the things!!! WOOOHOOO!!!!')
-49
View File
@@ -1,49 +0,0 @@
from os.path import join, dirname, abspath
from pythonforandroid.toolchain import Bootstrap
from setuptools import setup, find_packages
from bootstrap import LBRYServiceBootstrap
Bootstrap.bootstraps = {
'lbry-service': LBRYServiceBootstrap()
}
setup(
name='lbryservice',
version='0.1',
author="LBRY Inc.",
author_email="hello@lbry.io",
url="https://lbry.io",
description="Android Service for LBRY Network.",
license='MIT',
python_requires='>=3.7',
packages=find_packages(),
package_data={'service': ['*.py']},
options={
'apk': {
'dist_name': 'lbry-service',
'bootstrap': 'lbry-service',
'package': 'io.lbry.service',
'permissions': ['INTERNET'],
'requirements': ','.join([
# needed by aiohttp
'multidict', 'yarl', 'async_timeout', 'chardet',
# minimum needed by torba:
'aiohttp', 'coincurve', 'pbkdf2', 'cryptography', 'attrs',
abspath(join(dirname(__file__), '..', '..', 'torba')),
# minimum needed by lbrynet
'aioupnp', 'appdirs', 'distro', 'base58', 'jsonrpc', 'protobuf',
'msgpack', 'jsonschema', 'ecdsa', 'pyyaml', 'docopt',
abspath(join(dirname(__file__), '..')),
'genericndkbuild', 'pyjnius', 'sqlite3', 'python3'
]),
'android-api': '26',
'ndk-api': '21',
'ndk-version': 'r17c',
'arch': 'armeabi-v7a',
'sdk-dir': '/home/lex/projects/android',
'ndk-dir': '/home/lex/projects/android/android-ndk-r17c/'
}
}
)
+33
View File
@@ -0,0 +1,33 @@
$env:Path += ";C:\MinGW\bin\"
$env:Path += ";C:\Program Files (x86)\Windows Kits\10\bin\x86\"
gcc --version
mingw32-make --version
# build/install miniupnpc manually
tar zxf miniupnpc-1.9.tar.gz
cd miniupnpc-1.9
mingw32-make -f Makefile.mingw
python setupmingw32.py build --compiler=mingw32
python setupmingw32.py install
cd ..\
Remove-Item -Recurse -Force miniupnpc-1.9
# copy requirements from lbry, but remove miniupnpc (installed manually)
Get-Content ..\requirements.txt | Select-String -Pattern 'miniupnpc' -NotMatch | Out-File requirements_base.txt
python set_build.py
pip install -r requirements.txt
pip install ..\.
pyinstaller -y daemon.onefile.spec
pyinstaller -y cli.onefile.spec
pyinstaller -y console.onefile.spec
nuget install secure-file -ExcludeVersion
secure-file\tools\secure-file -decrypt .\lbry2.pfx.enc -secret "$env:pfx_key"
signtool.exe sign /f .\lbry2.pfx /p "$env:key_pass" /tr http://tsa.starfieldtech.com /td SHA256 /fd SHA256 dist\*.exe
python zip_daemon.py
python upload_assets.py
Executable
+59
View File
@@ -0,0 +1,59 @@
#!/bin/bash
set -euo pipefail
set -x
ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
cd "$ROOT"
BUILD_DIR="$ROOT/build"
FULL_BUILD="${FULL_BUILD:-false}"
if [ -n "${TEAMCITY_VERSION:-}" -o -n "${APPVEYOR:-}" ]; then
FULL_BUILD="true"
fi
[ -d "$BUILD_DIR/bulid" ] && rm -rf "$BUILD_DIR/build"
[ -d "$BUILD_DIR/dist" ] && rm -rf "$BUILD_DIR/dist"
if [ "$FULL_BUILD" == "true" ]; then
# install dependencies
$BUILD_DIR/prebuild.sh
VENV="$BUILD_DIR/venv"
if [ -d "$VENV" ]; then
rm -rf "$VENV"
fi
virtualenv "$VENV"
set +u
source "$VENV/bin/activate"
set -u
# must set build before installing lbrynet. otherwise it has no effect
python "$BUILD_DIR/set_build.py"
fi
cp "$ROOT/requirements.txt" "$BUILD_DIR/requirements_base.txt"
(
cd "$BUILD_DIR"
pip install -r requirements.txt
)
(
cd "$BUILD_DIR"
pyinstaller -y daemon.onefile.spec
pyinstaller -y cli.onefile.spec
pyinstaller -y console.onefile.spec
)
python "$BUILD_DIR/zip_daemon.py"
if [ "$FULL_BUILD" == "true" ]; then
# electron-build has a publish feature, but I had a hard time getting
# it to reliably work and it also seemed difficult to configure. Not proud of
# this, but it seemed better to write my own.
python "$BUILD_DIR/upload_assets.py"
deactivate
fi
echo 'Build complete.'
+42
View File
@@ -0,0 +1,42 @@
# -*- mode: python -*-
import platform
import os
dir = 'build';
cwd = os.getcwd()
if os.path.basename(cwd) != dir:
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
repo_base = os.path.abspath(os.path.join(cwd, '..'))
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
system = platform.system()
if system == 'Darwin':
icns = os.path.join(repo_base, 'build', 'icon.icns')
elif system == 'Linux':
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
elif system == 'Windows':
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
else:
print 'Warning: System {} has no icons'.format(system)
icns = None
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-cli', pathex=[cwd])
pyz = PYZ(a.pure, a.zipped_data)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
name='lbrynet-cli',
debug=False,
strip=False,
upx=True,
console=True,
icon=icns
)
+50
View File
@@ -0,0 +1,50 @@
# -*- mode: python -*-
import platform
import os
import lbryum
dir = 'build';
cwd = os.getcwd()
if os.path.basename(cwd) != dir:
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
repo_base = os.path.abspath(os.path.join(cwd, '..'))
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
system = platform.system()
if system == 'Darwin':
icns = os.path.join(repo_base, 'build', 'icon.icns')
elif system == 'Linux':
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
elif system == 'Windows':
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
else:
print 'Warning: System {} has no icons'.format(system)
icns = None
datas = [
(os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language + '.txt'), 'lbryum/wordlist')
for language in ('chinese_simplified', 'japanese', 'spanish','english', 'portuguese')
]
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-console', pathex=[cwd], datas=datas)
pyz = PYZ(a.pure, a.zipped_data)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
name='lbrynet-console',
debug=False,
strip=False,
upx=True,
console=True,
icon=icns
)
+50
View File
@@ -0,0 +1,50 @@
# -*- mode: python -*-
import platform
import os
import lbryum
dir = 'build';
cwd = os.getcwd()
if os.path.basename(cwd) != dir:
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
repo_base = os.path.abspath(os.path.join(cwd, '..'))
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
system = platform.system()
if system == 'Darwin':
icns = os.path.join(repo_base, 'build', 'icon.icns')
elif system == 'Linux':
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
elif system == 'Windows':
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
else:
print 'Warning: System {} has no icons'.format(system)
icns = None
datas = [
(os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language + '.txt'), 'lbryum/wordlist')
for language in ('chinese_simplified', 'japanese', 'spanish','english', 'portuguese')
]
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-daemon', pathex=[cwd], datas=datas)
pyz = PYZ(a.pure, a.zipped_data)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
name='lbrynet-daemon',
debug=False,
strip=False,
upx=True,
console=True,
icon=icns
)
+47
View File
@@ -0,0 +1,47 @@
# https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Setuptools-Entry-Point
def Entrypoint(dist, group, name,
scripts=None, pathex=None, binaries=None, datas=None,
hiddenimports=None, hookspath=None, excludes=None, runtime_hooks=None,
cipher=None, win_no_prefer_redirects=False, win_private_assemblies=False):
import pkg_resources
# get toplevel packages of distribution from metadata
def get_toplevel(dist):
distribution = pkg_resources.get_distribution(dist)
if distribution.has_metadata('top_level.txt'):
return list(distribution.get_metadata('top_level.txt').split())
else:
return []
hiddenimports = hiddenimports or []
packages = []
for distribution in hiddenimports:
packages += get_toplevel(distribution)
scripts = scripts or []
pathex = pathex or []
# get the entry point
ep = pkg_resources.get_entry_info(dist, group, name)
# insert path of the egg at the verify front of the search path
pathex = [ep.dist.location] + pathex
# script name must not be a valid module name to avoid name clashes on import
script_path = os.path.join(workpath, name + '-script.py')
print "creating script for entry point", dist, group, name
with open(script_path, 'w') as fh:
fh.write("import {0}\n".format(ep.module_name))
fh.write("{0}.{1}()\n".format(ep.module_name, '.'.join(ep.attrs)))
for package in packages:
fh.write("import {0}\n".format(package))
return Analysis([script_path] + scripts,
pathex=pathex,
binaries=binaries,
datas=datas,
hiddenimports=hiddenimports,
hookspath=hookspath,
excludes=excludes,
runtime_hooks=runtime_hooks,
cipher=cipher,
win_no_prefer_redirects=win_no_prefer_redirects,
win_private_assemblies=win_private_assemblies
)
File renamed without changes.

Before

Width:  |  Height:  |  Size: 7.4 KiB

After

Width:  |  Height:  |  Size: 7.4 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 6.1 KiB

After

Width:  |  Height:  |  Size: 6.1 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 97 KiB

After

Width:  |  Height:  |  Size: 97 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 361 KiB

After

Width:  |  Height:  |  Size: 361 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 5.3 KiB

After

Width:  |  Height:  |  Size: 5.3 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 31 KiB

After

Width:  |  Height:  |  Size: 31 KiB

Binary file not shown.
Binary file not shown.
+81
View File
@@ -0,0 +1,81 @@
#!/bin/bash
set -euo pipefail
set -x
LINUX=false
OSX=false
if [ "$(uname)" == "Darwin" ]; then
OSX=true
elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
LINUX=true
else
echo "Platform detection failed"
exit 1
fi
SUDO=''
if $LINUX && (( $EUID != 0 )); then
SUDO='sudo'
fi
cmd_exists() {
command -v "$1" >/dev/null 2>&1
return $?
}
set +eu
GITUSERNAME=$(git config --global --get user.name)
if [ -z "$GITUSERNAME" ]; then
git config --global user.name "$(whoami)"
fi
GITEMAIL=$(git config --global --get user.email)
if [ -z "$GITEMAIL" ]; then
git config --global user.email "$(whoami)@lbry.io"
fi
set -eu
if $LINUX; then
INSTALL="$SUDO apt-get install --no-install-recommends -y"
$INSTALL build-essential libssl-dev libffi-dev python2.7-dev wget
elif $OSX && ! cmd_exists brew ; then
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
fi
if ! cmd_exists python; then
if $LINUX; then
$INSTALL python2.7
elif $OSX; then
brew install python
fi
fi
PYTHON_VERSION=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
if [ "$PYTHON_VERSION" != "2.7" ]; then
echo "Python 2.7 required"
exit 1
fi
if ! cmd_exists pip; then
if $LINUX; then
$INSTALL python-pip
$SUDO pip install --upgrade pip
else
echo "Pip required"
exit 1
fi
fi
if $LINUX && [ "$(pip list --format=columns | grep setuptools | wc -l)" -ge 1 ]; then
#$INSTALL python-setuptools
$SUDO pip install setuptools
fi
if ! cmd_exists virtualenv; then
$SUDO pip install virtualenv
fi
+11
View File
@@ -0,0 +1,11 @@
# install daemon requirements (created by build script. see build.sh, build.ps1)
-r requirements_base.txt
# install daemon itself. make sure you run `pip install` from this dir. this is how you do relative file paths with pip
file:../.
# install other build requirements
PyInstaller==3.2.1
requests[security]==2.13.0
uritemplate==3.0.0
boto3==1.4.4
+3 -5
View File
@@ -1,4 +1,3 @@
"""Set the build version to be 'dev', 'qa', 'rc', 'release'"""
import os.path
@@ -16,12 +15,11 @@ def main():
def get_build():
try:
tag = subprocess.check_output(['git', 'describe', '--exact-match', '--all']).strip()
if re.match('tags\/v\d+\.\d+\.\d+rc\d+$', tag.decode()):
tag = subprocess.check_output(['git', 'describe', '--exact-match']).strip()
if re.match('v\d+\.\d+\.\d+rc\d+', tag):
return 'rc'
elif re.match('tags\/v\d+\.\d+\.\d+$', tag.decode()):
else:
return 'release'
return 'qa'
except subprocess.CalledProcessError:
# if the build doesn't have a tag
return 'qa'
+143
View File
@@ -0,0 +1,143 @@
import glob
import json
import os
import subprocess
import sys
import github
import uritemplate
import boto3
def main():
upload_to_github_if_tagged('lbryio/lbry')
upload_to_s3('daemon')
def get_asset_filename():
this_dir = os.path.dirname(os.path.realpath(__file__))
return glob.glob(this_dir + '/dist/*.zip')[0]
def upload_to_s3(folder):
tag = subprocess.check_output(['git', 'describe', '--always', '--abbrev=8', 'HEAD']).strip()
commit_date = subprocess.check_output([
'git', 'show', '-s', '--format=%cd', '--date=format:%Y%m%d-%H%I%S', 'HEAD']).strip()
asset_path = get_asset_filename()
bucket = 'releases.lbry.io'
key = folder + '/' + commit_date + '-' + tag + '/' + os.path.basename(asset_path)
print "Uploading " + asset_path + " to s3://" + bucket + '/' + key + ''
if 'AWS_ACCESS_KEY_ID' not in os.environ or 'AWS_SECRET_ACCESS_KEY' not in os.environ:
print 'Must set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to publish assets to s3'
return 1
s3 = boto3.resource(
's3',
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
config=boto3.session.Config(signature_version='s3v4')
)
s3.meta.client.upload_file(asset_path, bucket, key)
def upload_to_github_if_tagged(repo_name):
try:
current_tag = subprocess.check_output(
['git', 'describe', '--exact-match', 'HEAD']).strip()
except subprocess.CalledProcessError:
print 'Not uploading to GitHub as we are not currently on a tag'
return 1
print "Current tag: " + current_tag
if 'GH_TOKEN' not in os.environ:
print 'Must set GH_TOKEN in order to publish assets to a release'
return 1
gh_token = os.environ['GH_TOKEN']
auth = github.Github(gh_token)
repo = auth.get_repo(repo_name)
if not check_repo_has_tag(repo, current_tag):
print 'Tag {} is not in repo {}'.format(current_tag, repo)
# TODO: maybe this should be an error
return 1
asset_path = get_asset_filename()
print "Uploading " + asset_path + " to Github tag " + current_tag
release = get_github_release(repo, current_tag)
upload_asset_to_github(release, asset_path, gh_token)
def check_repo_has_tag(repo, target_tag):
tags = repo.get_tags().get_page(0)
for tag in tags:
if tag.name == target_tag:
return True
return False
def get_github_release(repo, current_tag):
for release in repo.get_releases():
if release.tag_name == current_tag:
return release
raise Exception('No release for {} was found'.format(current_tag))
def upload_asset_to_github(release, asset_to_upload, token):
basename = os.path.basename(asset_to_upload)
for asset in release.raw_data['assets']:
if asset['name'] == basename:
print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
return
upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
count = 0
while count < 10:
try:
output = _curl_uploader(upload_uri, asset_to_upload, token)
if 'errors' in output:
raise Exception(output)
else:
print 'Successfully uploaded to {}'.format(output['browser_download_url'])
except Exception:
print 'Failed uploading on attempt {}'.format(count + 1)
count += 1
def _curl_uploader(upload_uri, asset_to_upload, token):
# using requests.post fails miserably with SSL EPIPE errors. I spent
# half a day trying to debug before deciding to switch to curl.
#
# TODO: actually set the content type
print 'Using curl to upload {} to {}'.format(asset_to_upload, upload_uri)
cmd = [
'curl',
'-sS',
'-X', 'POST',
'-u', ':{}'.format(os.environ['GH_TOKEN']),
'--header', 'Content-Type: application/octet-stream',
'--data-binary', '@-',
upload_uri
]
# '-d', '{"some_key": "some_value"}',
print 'Calling curl:'
print cmd
print
with open(asset_to_upload, 'rb') as fp:
p = subprocess.Popen(cmd, stdin=fp, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
stdout, stderr = p.communicate()
print 'curl return code:', p.returncode
if stderr:
print 'stderr output from curl:'
print stderr
print 'stdout from curl:'
print stdout
return json.loads(stdout)
if __name__ == '__main__':
sys.exit(main())
+29
View File
@@ -0,0 +1,29 @@
import os
import platform
import subprocess
import sys
import zipfile
def main():
this_dir = os.path.dirname(os.path.realpath(__file__))
tag = subprocess.check_output(['git', 'describe']).strip()
zipfilename = 'lbrynet-daemon-{}-{}.zip'.format(tag, get_system_label())
full_filename = os.path.join(this_dir, 'dist', zipfilename)
executables = ['lbrynet-daemon', 'lbrynet-cli', 'lbrynet-console']
ext = '.exe' if platform.system() == 'Windows' else ''
with zipfile.ZipFile(full_filename, 'w') as myzip:
for executable in executables:
myzip.write(os.path.join(this_dir, 'dist', executable + ext), executable + ext)
def get_system_label():
system = platform.system()
if system == 'Darwin':
return 'macos'
else:
return system.lower()
if __name__ == '__main__':
sys.exit(main())
+3 -3
View File
@@ -100,7 +100,7 @@
<div class="md-flex__cell md-flex__cell--shrink">
<a href="/" title="LBRY" class="md-header-nav__button md-logo">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="24" height="24">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="24" height="24">
</a>
</div>
@@ -196,7 +196,7 @@
<label class="md-nav__title md-nav__title--site" for="drawer">
<span class="md-nav__button md-logo">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="48" height="48">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="48" height="48">
</span>
LBRY
@@ -304,4 +304,4 @@
</body>
</html>
</html>
-1605
View File
File diff suppressed because it is too large. Load diff
+95 -190
View File
@@ -307,13 +307,6 @@
blob_list
</a>
</li>
<li class="md-nav__item">
<a href="#blob_reflect" title="blob_reflect" class="md-nav__link">
blob_reflect
</a>
</li>
<li class="md-nav__item">
@@ -482,13 +475,6 @@
peer_list
</a>
</li>
<li class="md-nav__item">
<a href="#peer_ping" title="peer_ping" class="md-nav__link">
peer_ping
</a>
</li>
<li class="md-nav__item">
@@ -496,13 +482,6 @@
publish
</a>
</li>
<li class="md-nav__item">
<a href="#report_bug" title="report_bug" class="md-nav__link">
report_bug
</a>
</li>
<li class="md-nav__item">
@@ -730,13 +709,6 @@
blob_list
</a>
</li>
<li class="md-nav__item">
<a href="#blob_reflect" title="blob_reflect" class="md-nav__link">
blob_reflect
</a>
</li>
<li class="md-nav__item">
@@ -905,13 +877,6 @@
peer_list
</a>
</li>
<li class="md-nav__item">
<a href="#peer_ping" title="peer_ping" class="md-nav__link">
peer_ping
</a>
</li>
<li class="md-nav__item">
@@ -919,13 +884,6 @@
publish
</a>
</li>
<li class="md-nav__item">
<a href="#report_bug" title="report_bug" class="md-nav__link">
report_bug
</a>
</li>
<li class="md-nav__item">
@@ -1113,16 +1071,18 @@
<pre><code class="text">Announce blobs to the DHT
Usage:
blob_announce [&lt;blob_hash&gt; | --blob_hash=&lt;blob_hash&gt;]
[&lt;stream_hash&gt; | --stream_hash=&lt;stream_hash&gt;] | [&lt;sd_hash&gt; | --sd_hash=&lt;sd_hash&gt;]
blob_announce [--announce_all] [&lt;blob_hash&gt; | --blob_hash=&lt;blob_hash&gt;]
[&lt;stream_hash&gt; | --stream_hash=&lt;stream_hash&gt;]
[&lt;sd_hash&gt; | --sd_hash=&lt;sd_hash&gt;]
Options:
--blob_hash=&lt;blob_hash&gt; : (str) announce a blob, specified by blob_hash
--stream_hash=&lt;stream_hash&gt; : (str) announce all blobs associated with
stream_hash
--sd_hash=&lt;sd_hash&gt; : (str) announce all blobs associated with
sd_hash and the sd_hash itself
--announce_all=&lt;announce_all&gt; : (bool) announce all the blobs possessed by user
--blob_hash=&lt;blob_hash&gt; : (str) announce a blob, specified by blob_hash
--stream_hash=&lt;stream_hash&gt; : (str) announce all blobs associated with
stream_hash
--sd_hash=&lt;sd_hash&gt; : (str) announce all blobs associated with
sd_hash and the sd_hash itself
Returns:
(bool) true if successful
@@ -1212,20 +1172,6 @@ Returns:
(list) List of blob hashes
</code></pre>
<h2 id="blob_reflect">blob_reflect</h2>
<pre><code class="text">Reflects specified blobs
Usage:
blob_reflect (&lt;blob_hashes&gt;...) [--reflector_server=&lt;reflector_server&gt;]
Options:
--reflector_server=&lt;reflector_server&gt; : (str) reflector address
Returns:
(list) reflected blob hashes
</code></pre>
<h2 id="blob_reflect_all">blob_reflect_all</h2>
<pre><code class="text">Reflects all saved blobs
@@ -1273,7 +1219,8 @@ Returns:
<pre><code class="text">Import serialized channel signing information (to allow signing new claims to the channel)
Usage:
channel_import (&lt;serialized_certificate_info&gt; | --serialized_certificate_info=&lt;serialized_certificate_info&gt;)
channel_import (&lt;serialized_certificate_info&gt; |
--serialized_certificate_info=&lt;serialized_certificate_info&gt;)
Options:
@@ -1454,7 +1401,7 @@ Returns:
'is_spent': (bool) true if claim is abandoned, false otherwise
'name': (str) name of the claim
'permanent_url': (str) permanent url of the claim,
'txid': (str) txid of the claim
'txid': (str) txid of the cliam
'nout': (int) nout of the claim
'value': (str) value of the claim
},
@@ -1579,12 +1526,12 @@ Usage:
Options:
--a_arg : (bool) a arg
--b_arg : (bool) b arg
--pos_arg=&lt;pos_arg&gt; : (int) pos arg
--pos_args=&lt;pos_args&gt; : (int) pos args
--pos_arg2=&lt;pos_arg2&gt; : (int) pos arg 2
--pos_arg3=&lt;pos_arg3&gt; : (int) pos arg 3
--a_arg : a arg
--b_arg : b arg
--pos_arg=&lt;pos_arg&gt; : pos arg
--pos_args=&lt;pos_args&gt; : pos args
--pos_arg2=&lt;pos_arg2&gt; : pos arg 2
--pos_arg3=&lt;pos_arg3&gt; : pos arg 3
Returns:
pos args
@@ -1656,7 +1603,7 @@ Usage:
file_list [--sd_hash=&lt;sd_hash&gt;] [--file_name=&lt;file_name&gt;] [--stream_hash=&lt;stream_hash&gt;]
[--rowid=&lt;rowid&gt;] [--claim_id=&lt;claim_id&gt;] [--outpoint=&lt;outpoint&gt;] [--txid=&lt;txid&gt;] [--nout=&lt;nout&gt;]
[--channel_claim_id=&lt;channel_claim_id&gt;] [--channel_name=&lt;channel_name&gt;]
[--claim_name=&lt;claim_name&gt;] [--full_status] [--sort=&lt;sort_method&gt;...]
[--claim_name=&lt;claim_name&gt;] [--full_status]
Options:
@@ -1674,9 +1621,6 @@ Options:
--claim_name=&lt;claim_name&gt; : (str) get file with matching claim name
--full_status : (bool) full status, populate the
'message' and 'size' fields
--sort=&lt;sort_method&gt; : (str) sort by any property, like 'file_name'
or 'metadata.author'; to specify direction
append ',asc' or ',desc'
Returns:
(list) List of files
@@ -1824,23 +1768,7 @@ Options:
--timeout=&lt;timeout&gt; : (int) peer search timeout in seconds
Returns:
(list) List of contact dictionaries {'host': &lt;peer ip&gt;, 'port': &lt;peer port&gt;, 'node_id': &lt;peer node id&gt;}
</code></pre>
<h2 id="peer_ping">peer_ping</h2>
<pre><code class="text">Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,
if not provided the peer is located first.
Usage:
peer_ping (&lt;node_id&gt; | --node_id=&lt;node_id&gt;) [&lt;address&gt; | --address=&lt;address&gt;] [&lt;port&gt; | --port=&lt;port&gt;]
Options:
--address=&lt;address&gt; : (str) ip address of the peer
--port=&lt;port&gt; : (int) udp port of the peer
Returns:
(str) pong, or {'error': &lt;error message&gt;} if an error is encountered
(list) List of contacts
</code></pre>
<h2 id="publish">publish</h2>
@@ -1870,40 +1798,40 @@ Usage:
Options:
--name=&lt;name&gt; : (str) name of the content
--bid=&lt;bid&gt; : (decimal) amount to back the claim
--metadata=&lt;metadata&gt; : (dict) ClaimDict to associate with the claim.
--file_path=&lt;file_path&gt; : (str) path to file to be associated with name. If provided,
a lbry stream of this file will be used in 'sources'.
If no path is given but a sources dict is provided,
it will be used. If neither are provided, an
error is raised.
--fee=&lt;fee&gt; : (dict) Dictionary representing key fee to download content:
{
'currency': currency_symbol,
'amount': decimal,
'address': str, optional
}
supported currencies: LBC, USD, BTC
If an address is not provided a new one will be
automatically generated. Default fee is zero.
--title=&lt;title&gt; : (str) title of the publication
--description=&lt;description&gt; : (str) description of the publication
--author=&lt;author&gt; : (str) author of the publication
--language=&lt;language&gt; : (str) language of the publication
--license=&lt;license&gt; : (str) publication license
--license_url=&lt;license_url&gt; : (str) publication license url
--thumbnail=&lt;thumbnail&gt; : (str) thumbnail url
--preview=&lt;preview&gt; : (str) preview url
--nsfw=&lt;nsfw&gt; : (bool) whether the content is nsfw
--sources=&lt;sources&gt; : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
--channel_name=&lt;channel_name&gt; : (str) name of the publisher channel name in the wallet
--channel_id=&lt;channel_id&gt; : (str) claim id of the publisher channel, does not check
for channel claim being in the wallet. This allows
publishing to a channel where only the certificate
private key is in the wallet.
--claim_address=&lt;claim_address&gt; : (str) address where the claim is sent to, if not specified
new address will automatically be created
--name=&lt;name&gt; : (str) name of the content
--bid=&lt;bid&gt; : (float) amount to back the claim
--metadata=&lt;metadata&gt; : (dict) ClaimDict to associate with the claim.
--file_path=&lt;file_path&gt; : (str) path to file to be associated with name. If provided,
a lbry stream of this file will be used in 'sources'.
If no path is given but a sources dict is provided,
it will be used. If neither are provided, an
error is raised.
--fee=&lt;fee&gt; : (dict) Dictionary representing key fee to download content:
{
'currency': currency_symbol,
'amount': float,
'address': str, optional
}
supported currencies: LBC, USD, BTC
If an address is not provided a new one will be
automatically generated. Default fee is zero.
--title=&lt;title&gt; : (str) title of the publication
--description=&lt;description&gt; : (str) description of the publication
--author=&lt;author&gt; : (str) author of the publication
--language=&lt;language&gt; : (str) language of the publication
--license=&lt;license&gt; : (str) publication license
--license_url=&lt;license_url&gt; : (str) publication license url
--thumbnail=&lt;thumbnail&gt; : (str) thumbnail url
--preview=&lt;preview&gt; : (str) preview url
--nsfw=&lt;nsfw&gt; : (bool) title of the publication
--sources=&lt;sources&gt; : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
--channel_name=&lt;channel_name&gt; : (str) name of the publisher channel name in the wallet
--channel_id=&lt;channel_id&gt; : (str) claim id of the publisher channel, does not check
for channel claim being in the wallet. This allows
publishing to a channel where only the certificate
private key is in the wallet.
--claim_address=&lt;claim_address&gt; : (str) address where the claim is sent to, if not specified
new address wil automatically be created
Returns:
(dict) Dictionary containing result of the claim
@@ -1911,25 +1839,11 @@ Returns:
'tx' : (str) hex encoded transaction
'txid' : (str) txid of resulting claim
'nout' : (int) nout of the resulting claim
'fee' : (decimal) fee paid for the claim transaction
'fee' : (float) fee paid for the claim transaction
'claim_id' : (str) claim ID of the resulting claim
}
</code></pre>
<h2 id="report_bug">report_bug</h2>
<pre><code class="text">Report a bug to slack
Usage:
report_bug (&lt;message&gt; | --message=&lt;message&gt;)
Options:
--message=&lt;message&gt; : (str) Description of the bug
Returns:
(bool) true if successful
</code></pre>
<h2 id="resolve">resolve</h2>
<pre><code class="text">Resolve given LBRY URIs
@@ -2031,7 +1945,6 @@ Returns:
&lt;bucket index&gt;: [
{
&quot;address&quot;: (str) peer address,
&quot;port&quot;: (int) peer udp port
&quot;node_id&quot;: (str) peer node id,
&quot;blobs&quot;: (list) blob hashes announced by peer
}
@@ -2112,62 +2025,54 @@ Returns:
<pre><code class="text">Get daemon status
Usage:
status
status [--session_status] [--dht_status]
Options:
None
--session_status : (bool) include session status in results
--dht_status : (bool) include dht network and peer status
Returns:
(dict) lbrynet-daemon status
{
'installation_id': (str) installation id - base58,
'is_running': (bool),
'lbry_id': lbry peer id, base58,
'installation_id': installation id, base58,
'is_running': bool,
'is_first_run': bool,
'skipped_components': (list) [names of skipped components (str)],
'startup_status': { Does not include components which have been skipped
'database': (bool),
'wallet': (bool),
'session': (bool),
'dht': (bool),
'hash_announcer': (bool),
'stream_identifier': (bool),
'file_manager': (bool),
'blob_manager': (bool),
'blockchain_headers': (bool),
'peer_protocol_server': (bool),
'reflector': (bool),
'upnp': (bool),
'exchange_rate_manager': (bool),
'startup_status': {
'code': status code,
'message': status message
},
'connection_status': {
'code': (str) connection status code,
'message': (str) connection status message
'code': connection status code,
'message': connection status message
},
'blockchain_headers': {
'downloading_headers': (bool),
'download_progress': (float) 0-100.0
'blockchain_status': {
'blocks': local blockchain height,
'blocks_behind': remote_height - local_height,
'best_blockhash': block hash of most recent block,
},
'wallet': {
'blocks': (int) local blockchain height,
'blocks_behind': (int) remote_height - local_height,
'best_blockhash': (str) block hash of most recent block,
'is_encrypted': (bool),
'is_locked': (bool),
},
'dht': {
'node_id': (str) lbry dht node id - hex encoded,
'peers_in_routing_table': (int) the number of peers in the routing table,
},
'blob_manager': {
'finished_blobs': (int) number of finished blobs in the blob manager,
},
'hash_announcer': {
'announce_queue_size': (int) number of blobs currently queued to be announced
},
'file_manager': {
'managed_files': (int) count of files in the file manager,
}
'wallet_is_encrypted': bool,
If given the session status option:
'session_status': {
'managed_blobs': count of blobs in the blob manager,
'managed_streams': count of streams in the file manager
'announce_queue_size': number of blobs currently queued to be announced
'should_announce_blobs': number of blobs that should be announced
}
If given the dht status option:
'dht_status': {
'kbps_received': current kbps receiving,
'kbps_sent': current kdps being sent,
'total_bytes_sent': total bytes sent,
'total_bytes_received': total bytes received,
'queries_received': number of queries received per second,
'queries_sent': number of queries sent per second,
'recent_contacts': count of recently contacted peers,
'unique_contacts': count of unique peers
},
}
</code></pre>
@@ -2184,7 +2089,7 @@ Options:
--uri=&lt;uri&gt; : (str) check availability for this uri
--search_timeout=&lt;search_timeout&gt; : (int) how long to search for peers for the blob
in the dht
--blob_timeout=&lt;blob_timeout&gt; : (int) how long to try downloading from a peer
--search_timeout=&lt;blob_timeout&gt; : (int) how long to try downloading from a peer
Returns:
(dict) {
@@ -2475,9 +2380,9 @@ Usage:
Options:
--amount=&lt;amount&gt; : (Decimal) amount of credit to send
--address=&lt;address&gt; : (str) address to send credits to
--claim_id=&lt;claim_id&gt; : (str) claim_id of the claim to send to tip to
--amount=&lt;amount&gt; : (float) amount of credit to send
--address=&lt;address&gt; : (str) address to send credits to
--claim_id=&lt;claim_id&gt; : (float) claim_id of the claim to send to tip to
Returns:
If sending to an address:
+89 -174
View File
@@ -295,13 +295,6 @@
blob_list
</a>
</li>
<li class="md-nav__item">
<a href="#blob_reflect" title="blob_reflect" class="md-nav__link">
blob_reflect
</a>
</li>
<li class="md-nav__item">
@@ -470,13 +463,6 @@
peer_list
</a>
</li>
<li class="md-nav__item">
<a href="#peer_ping" title="peer_ping" class="md-nav__link">
peer_ping
</a>
</li>
<li class="md-nav__item">
@@ -484,13 +470,6 @@
publish
</a>
</li>
<li class="md-nav__item">
<a href="#report_bug" title="report_bug" class="md-nav__link">
report_bug
</a>
</li>
<li class="md-nav__item">
@@ -730,13 +709,6 @@
blob_list
</a>
</li>
<li class="md-nav__item">
<a href="#blob_reflect" title="blob_reflect" class="md-nav__link">
blob_reflect
</a>
</li>
<li class="md-nav__item">
@@ -905,13 +877,6 @@
peer_list
</a>
</li>
<li class="md-nav__item">
<a href="#peer_ping" title="peer_ping" class="md-nav__link">
peer_ping
</a>
</li>
<li class="md-nav__item">
@@ -919,13 +884,6 @@
publish
</a>
</li>
<li class="md-nav__item">
<a href="#report_bug" title="report_bug" class="md-nav__link">
report_bug
</a>
</li>
<li class="md-nav__item">
@@ -1113,11 +1071,12 @@
<pre><code class="text">Announce blobs to the DHT
Args:
'blob_hash' : (str) announce a blob, specified by blob_hash
'stream_hash' : (str) announce all blobs associated with
stream_hash
'sd_hash' : (str) announce all blobs associated with
sd_hash and the sd_hash itself
'announce_all' : (bool) announce all the blobs possessed by user
'blob_hash' : (str) announce a blob, specified by blob_hash
'stream_hash' : (str) announce all blobs associated with
stream_hash
'sd_hash' : (str) announce all blobs associated with
sd_hash and the sd_hash itself
Returns:
(bool) true if successful
@@ -1185,16 +1144,6 @@ Returns:
(list) List of blob hashes
</code></pre>
<h2 id="blob_reflect">blob_reflect</h2>
<pre><code class="text">Reflects specified blobs
Args:
'reflector_server' : (str) reflector address
Returns:
(list) reflected blob hashes
</code></pre>
<h2 id="blob_reflect_all">blob_reflect_all</h2>
<pre><code class="text">Reflects all saved blobs
@@ -1230,7 +1179,7 @@ Returns:
<pre><code class="text">Import serialized channel signing information (to allow signing new claims to the channel)
Args:
'serialized_certificate_info' (required) : (str) certificate info
'serialized_certificate_info' : (str) certificate info
Returns:
(dict) Result dictionary
@@ -1380,7 +1329,7 @@ Returns:
'is_spent': (bool) true if claim is abandoned, false otherwise
'name': (str) name of the claim
'permanent_url': (str) permanent url of the claim,
'txid': (str) txid of the claim
'txid': (str) txid of the cliam
'nout': (int) nout of the claim
'value': (str) value of the claim
},
@@ -1479,12 +1428,12 @@ Returns:
<h2 id="cli_test_command">cli_test_command</h2>
<pre><code class="text">This command is only for testing the CLI argument parsing
Args:
'a_arg' : (bool) a arg
'b_arg' : (bool) b arg
'pos_arg' (required) : (int) pos arg
'pos_args' : (int) pos args
'pos_arg2' : (int) pos arg 2
'pos_arg3' : (int) pos arg 3
'a_arg' : a arg
'b_arg' : b arg
'pos_arg' (required) : pos arg
'pos_args' : pos args
'pos_arg2' : pos arg 2
'pos_arg3' : pos arg 3
Returns:
pos args
@@ -1552,9 +1501,6 @@ Args:
'claim_name' : (str) get file with matching claim name
'full_status' : (bool) full status, populate the
'message' and 'size' fields
'sort' : (str) sort by any property, like 'file_name'
or 'metadata.author'; to specify direction
append ',asc' or ',desc'
Returns:
(list) List of files
@@ -1678,19 +1624,7 @@ Args:
'timeout' : (int) peer search timeout in seconds
Returns:
(list) List of contact dictionaries {'host': &lt;peer ip&gt;, 'port': &lt;peer port&gt;, 'node_id': &lt;peer node id&gt;}
</code></pre>
<h2 id="peer_ping">peer_ping</h2>
<pre><code class="text">Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,
if not provided the peer is located first.
Args:
'address' : (str) ip address of the peer
'port' : (int) udp port of the peer
Returns:
(str) pong, or {'error': &lt;error message&gt;} if an error is encountered
(list) List of contacts
</code></pre>
<h2 id="publish">publish</h2>
@@ -1710,40 +1644,40 @@ fee, title, description, author, language, license, license_url, thumbnail, prev
or sources. Individual arguments will overwrite the fields specified in metadata argument.
Args:
'name' (required) : (str) name of the content
'bid' (required) : (decimal) amount to back the claim
'metadata' : (dict) ClaimDict to associate with the claim.
'file_path' : (str) path to file to be associated with name. If provided,
a lbry stream of this file will be used in 'sources'.
If no path is given but a sources dict is provided,
it will be used. If neither are provided, an
error is raised.
'fee' : (dict) Dictionary representing key fee to download content:
{
'currency': currency_symbol,
'amount': decimal,
'address': str, optional
}
supported currencies: LBC, USD, BTC
If an address is not provided a new one will be
automatically generated. Default fee is zero.
'title' : (str) title of the publication
'description' : (str) description of the publication
'author' : (str) author of the publication
'language' : (str) language of the publication
'license' : (str) publication license
'license_url' : (str) publication license url
'thumbnail' : (str) thumbnail url
'preview' : (str) preview url
'nsfw' : (bool) whether the content is nsfw
'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
'channel_name' : (str) name of the publisher channel name in the wallet
'channel_id' : (str) claim id of the publisher channel, does not check
for channel claim being in the wallet. This allows
publishing to a channel where only the certificate
private key is in the wallet.
'claim_address' : (str) address where the claim is sent to, if not specified
new address will automatically be created
'name' (required) : (str) name of the content
'bid' (required) : (float) amount to back the claim
'metadata' : (dict) ClaimDict to associate with the claim.
'file_path' : (str) path to file to be associated with name. If provided,
a lbry stream of this file will be used in 'sources'.
If no path is given but a sources dict is provided,
it will be used. If neither are provided, an
error is raised.
'fee' : (dict) Dictionary representing key fee to download content:
{
'currency': currency_symbol,
'amount': float,
'address': str, optional
}
supported currencies: LBC, USD, BTC
If an address is not provided a new one will be
automatically generated. Default fee is zero.
'title' : (str) title of the publication
'description' : (str) description of the publication
'author' : (str) author of the publication
'language' : (str) language of the publication
'license' : (str) publication license
'license_url' : (str) publication license url
'thumbnail' : (str) thumbnail url
'preview' : (str) preview url
'nsfw' : (bool) title of the publication
'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
'channel_name' : (str) name of the publisher channel name in the wallet
'channel_id' : (str) claim id of the publisher channel, does not check
for channel claim being in the wallet. This allows
publishing to a channel where only the certificate
private key is in the wallet.
'claim_address' : (str) address where the claim is sent to, if not specified
new address wil automatically be created
Returns:
(dict) Dictionary containing result of the claim
@@ -1751,21 +1685,11 @@ Returns:
'tx' : (str) hex encoded transaction
'txid' : (str) txid of resulting claim
'nout' : (int) nout of the resulting claim
'fee' : (decimal) fee paid for the claim transaction
'fee' : (float) fee paid for the claim transaction
'claim_id' : (str) claim ID of the resulting claim
}
</code></pre>
<h2 id="report_bug">report_bug</h2>
<pre><code class="text">Report a bug to slack
Args:
'message' (required) : (str) Description of the bug
Returns:
(bool) true if successful
</code></pre>
<h2 id="resolve">resolve</h2>
<pre><code class="text">Resolve given LBRY URIs
@@ -1855,7 +1779,6 @@ Returns:
&lt;bucket index&gt;: [
{
&quot;address&quot;: (str) peer address,
&quot;port&quot;: (int) peer udp port
&quot;node_id&quot;: (str) peer node id,
&quot;blobs&quot;: (list) blob hashes announced by peer
}
@@ -1915,58 +1838,50 @@ Returns:
<pre><code class="text">Get daemon status
Args:
None
'session_status' : (bool) include session status in results
'dht_status' : (bool) include dht network and peer status
Returns:
(dict) lbrynet-daemon status
{
'installation_id': (str) installation id - base58,
'is_running': (bool),
'lbry_id': lbry peer id, base58,
'installation_id': installation id, base58,
'is_running': bool,
'is_first_run': bool,
'skipped_components': (list) [names of skipped components (str)],
'startup_status': { Does not include components which have been skipped
'database': (bool),
'wallet': (bool),
'session': (bool),
'dht': (bool),
'hash_announcer': (bool),
'stream_identifier': (bool),
'file_manager': (bool),
'blob_manager': (bool),
'blockchain_headers': (bool),
'peer_protocol_server': (bool),
'reflector': (bool),
'upnp': (bool),
'exchange_rate_manager': (bool),
'startup_status': {
'code': status code,
'message': status message
},
'connection_status': {
'code': (str) connection status code,
'message': (str) connection status message
'code': connection status code,
'message': connection status message
},
'blockchain_headers': {
'downloading_headers': (bool),
'download_progress': (float) 0-100.0
'blockchain_status': {
'blocks': local blockchain height,
'blocks_behind': remote_height - local_height,
'best_blockhash': block hash of most recent block,
},
'wallet': {
'blocks': (int) local blockchain height,
'blocks_behind': (int) remote_height - local_height,
'best_blockhash': (str) block hash of most recent block,
'is_encrypted': (bool),
'is_locked': (bool),
},
'dht': {
'node_id': (str) lbry dht node id - hex encoded,
'peers_in_routing_table': (int) the number of peers in the routing table,
},
'blob_manager': {
'finished_blobs': (int) number of finished blobs in the blob manager,
},
'hash_announcer': {
'announce_queue_size': (int) number of blobs currently queued to be announced
},
'file_manager': {
'managed_files': (int) count of files in the file manager,
}
'wallet_is_encrypted': bool,
If given the session status option:
'session_status': {
'managed_blobs': count of blobs in the blob manager,
'managed_streams': count of streams in the file manager
'announce_queue_size': number of blobs currently queued to be announced
'should_announce_blobs': number of blobs that should be announced
}
If given the dht status option:
'dht_status': {
'kbps_received': current kbps receiving,
'kbps_sent': current kdps being sent,
'total_bytes_sent': total bytes sent,
'total_bytes_received': total bytes received,
'queries_received': number of queries received per second,
'queries_sent': number of queries sent per second,
'recent_contacts': count of recently contacted peers,
'unique_contacts': count of unique peers
},
}
</code></pre>
@@ -1977,7 +1892,7 @@ Args:
'uri' (required) : (str) check availability for this uri
'search_timeout' : (int) how long to search for peers for the blob
in the dht
'blob_timeout' : (int) how long to try downloading from a peer
'search_timeout' : (int) how long to try downloading from a peer
Returns:
(dict) {
@@ -2209,9 +2124,9 @@ to the owner of a claim specified by uri. A tip is a claim support where the rec
of the support is the claim address for the claim being supported.
Args:
'amount' (required) : (Decimal) amount of credit to send
'address' (required) : (str) address to send credits to
'claim_id' (required) : (str) claim_id of the claim to send to tip to
'amount' (required) : (float) amount of credit to send
'address' (required) : (str) address to send credits to
'claim_id' (required) : (float) claim_id of the claim to send to tip to
Returns:
If sending to an address:
+24 -54
View File
@@ -2,7 +2,7 @@
"docs": [
{
"location": "/",
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'announce_all' : (bool) announce all the blobs possessed by user\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State of claims assigned for the name\n {\n 'claims': (list) list of claims for the Line truncated
"title": "API"
},
{
@@ -12,7 +12,7 @@
},
{
"location": "/#blob_announce",
"text": "Announce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful",
"text": "Announce blobs to the DHT\n\nArgs:\n 'announce_all' : (bool) announce all the blobs possessed by user\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful",
"title": "blob_announce"
},
{
@@ -35,11 +35,6 @@
"text": "Returns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes",
"title": "blob_list"
},
{
"location": "/#blob_reflect",
"text": "Reflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes",
"title": "blob_reflect"
},
{
"location": "/#blob_reflect_all",
"text": "Reflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful",
@@ -57,7 +52,7 @@
},
{
"location": "/#channel_import",
"text": "Import serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary",
"text": "Import serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' : (str) certificate info\n\nReturns:\n (dict) Result dictionary",
"title": "channel_import"
},
{
@@ -87,7 +82,7 @@
},
{
"location": "/#claim_list_mine",
"text": "List my name claims\n\nArgs:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the claim\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
"text": "List my name claims\n\nArgs:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the cliam\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
"title": "claim_list_mine"
},
{
@@ -112,7 +107,7 @@
},
{
"location": "/#cli_test_command",
"text": "This command is only for testing the CLI argument parsing\nArgs:\n 'a_arg' : (bool) a arg\n 'b_arg' : (bool) b arg\n 'pos_arg' (required) : (int) pos arg\n 'pos_args' : (int) pos args\n 'pos_arg2' : (int) pos arg 2\n 'pos_arg3' : (int) pos arg 3\n\nReturns:\n pos args",
"text": "This command is only for testing the CLI argument parsing\nArgs:\n 'a_arg' : a arg\n 'b_arg' : b arg\n 'pos_arg' (required) : pos arg\n 'pos_args' : pos args\n 'pos_arg2' : pos arg 2\n 'pos_arg3' : pos arg 3\n\nReturns:\n pos args",
"title": "cli_test_command"
},
{
@@ -132,7 +127,7 @@
},
{
"location": "/#file_list",
"text": "List files limited by optional filters\n\nArgs:\n 'sd_hash' : (str) get file with matching sd hash\n 'file_name' : (str) get file with matching file name in the\n downloads folder\n 'stream_hash' : (str) get file with matching stream hash\n 'rowid' : (int) get file with matching row id\n 'claim_id' : (str) get file with matching claim id\n 'outpoint' : (str) get file with matching claim outpoint\n 'txid' : (str) get file with matching claim txid\n 'nout' : (int) get file with matching claim nout\n 'channel_claim_id' : (str) get file with matching channel claim id\n 'channel_name' : (str) get file with matching channel name\n 'claim_name' : (str) get file with matching claim name\n 'full_status' : (bool) full status, populate the\n 'message' and 'size' fields\n 'sort' : (str) sort by any property, like 'file_name'\n or 'metadata.author'; to specify direction\n append ',asc' or ',desc'\n\nReturns:\n (list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes, None if full_status is false,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) num_completed, None if full_status is false,\n 'blobs_in_stream': (int) None if full_status is false,\n 'status': (str) downloader status, None if full_status is false,\n 'claim_id': (str) None if full_status is false or if claim is not found,\n 'outpoint': (str) None if full_status is false or if claim is not found,\n 'txid': (str) None if full_status is false or if claim is not found,\n 'nout': (int) None if full_status is false or if claim is not found,\n 'metadata': (dict) None if full_status is false or if claim is not found,\n 'channel_claim_id': (str) None if full_status is false or if claim is not found or signed,\n 'channel_name': (str) None if full_status is false or if claim is not found or signed,\n 'claim_name': (str) None if full_status is false or if claim is not found\n },\n ]",
"text": "List files limited by optional filters\n\nArgs:\n 'sd_hash' : (str) get file with matching sd hash\n 'file_name' : (str) get file with matching file name in the\n downloads folder\n 'stream_hash' : (str) get file with matching stream hash\n 'rowid' : (int) get file with matching row id\n 'claim_id' : (str) get file with matching claim id\n 'outpoint' : (str) get file with matching claim outpoint\n 'txid' : (str) get file with matching claim txid\n 'nout' : (int) get file with matching claim nout\n 'channel_claim_id' : (str) get file with matching channel claim id\n 'channel_name' : (str) get file with matching channel name\n 'claim_name' : (str) get file with matching claim name\n 'full_status' : (bool) full status, populate the\n 'message' and 'size' fields\n\nReturns:\n (list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes, None if full_status is false,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) num_completed, None if full_status is false,\n 'blobs_in_stream': (int) None if full_status is false,\n 'status': (str) downloader status, None if full_status is false,\n 'claim_id': (str) None if full_status is false or if claim is not found,\n 'outpoint': (str) None if full_status is false or if claim is not found,\n 'txid': (str) None if full_status is false or if claim is not found,\n 'nout': (int) None if full_status is false or if claim is not found,\n 'metadata': (dict) None if full_status is false or if claim is not found,\n 'channel_claim_id': (str) None if full_status is false or if claim is not found or signed,\n 'channel_name': (str) None if full_status is false or if claim is not found or signed,\n 'claim_name': (str) None if full_status is false or if claim is not found\n },\n ]",
"title": "file_list"
},
{
@@ -157,24 +152,14 @@
},
{
"location": "/#peer_list",
"text": "Get peers for blob hash\n\nArgs:\n 'blob_hash' (required) : (str) find available peers for this blob hash\n 'timeout' : (int) peer search timeout in seconds\n\nReturns:\n (list) List of contact dictionaries {'host': peer ip , 'port': peer port , 'node_id': peer node id }",
"text": "Get peers for blob hash\n\nArgs:\n 'blob_hash' (required) : (str) find available peers for this blob hash\n 'timeout' : (int) peer search timeout in seconds\n\nReturns:\n (list) List of contacts",
"title": "peer_list"
},
{
"location": "/#peer_ping",
"text": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.\n\nArgs:\n 'address' : (str) ip address of the peer\n 'port' : (int) udp port of the peer\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
"title": "peer_ping"
},
{
"location": "/#publish",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (float) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': float,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) title of the publication\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"title": "publish"
},
{
"location": "/#report_bug",
"text": "Report a bug to slack\n\nArgs:\n 'message' (required) : (str) Description of the bug\n\nReturns:\n (bool) true if successful",
"title": "report_bug"
},
{
"location": "/#resolve",
"text": "Resolve given LBRY URIs\n\nArgs:\n 'force' : (bool) force refresh and ignore cache\n 'uri' (required) : (str) uri to resolve\n 'uris' : (list) uris to resolve\n\nReturns:\n Dictionary of results, keyed by uri\n ' uri ': {\n If a resolution error occurs:\n 'error': Error message\n\n If the uri resolves to a channel or a claim in a channel:\n 'certificate': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the certificate claim,\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}],\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n\n If the uri resolves to a channel:\n 'claims_in_channel': (int) number of claims in the channel,\n\n If the uri resolves to a claim:\n 'claim': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the claim,\n 'channel_name': (str) channel name if claim is in a channel\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}]\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n }",
@@ -187,7 +172,7 @@
},
{
"location": "/#routing_table_get",
"text": "Get DHT routing information\n\nArgs:\n None\n\nReturns:\n (dict) dictionary containing routing and contact information\n {\n buckets : {\n bucket index : [\n {\n address : (str) peer address,\n port : (int) peer udp port\n node_id : (str) peer node id,\n blobs : (list) blob hashes announced by peer\n }\n ]\n },\n contacts : (list) contact node ids,\n blob_hashes : (list) all of the blob hashes stored by peers in the list of buckets,\n node_id : (str) the local dht node id\n }",
"text": "Get DHT routing information\n\nArgs:\n None\n\nReturns:\n (dict) dictionary containing routing and contact information\n {\n buckets : {\n bucket index : [\n {\n address : (str) peer address,\n node_id : (str) peer node id,\n blobs : (list) blob hashes announced by peer\n }\n ]\n },\n contacts : (list) contact node ids,\n blob_hashes : (list) all of the blob hashes stored by peers in the list of buckets,\n node_id : (str) the local dht node id\n }",
"title": "routing_table_get"
},
{
@@ -202,12 +187,12 @@
},
{
"location": "/#status",
"text": "Get daemon status\n\nArgs:\n None\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }",
"text": "Get daemon status\n\nArgs:\n 'session_status' : (bool) include session status in results\n 'dht_status' : (bool) include dht network and peer status\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'lbry_id': lbry peer id, base58,\n 'installation_id': installation id, base58,\n 'is_running': bool,\n 'is_first_run': bool,\n 'startup_status': {\n 'code': status code,\n 'message': status message\n },\n 'connection_status': {\n 'code': connection status code,\n 'message': connection status message\n },\n 'blockchain_status': {\n 'blocks': local blockchain height,\n 'blocks_behind': remote_height - local_height,\n 'best_blockhash': block hash of most recent block,\n },\n 'wallet_is_encrypted': bool,\n\n If given the session status option:\n 'session_status': {\n 'managed_blobs': count of blobs in the blob manager,\n 'managed_streams': count of streams in the file manager\n 'announce_queue_size': number of blobs currently queued to be announced\n 'should_announce_blobs': number of blobs that should be announced\n }\n\n If given the dht status option:\n 'dht_status': {\n 'kbps_received': current kbps receiving,\n 'kbps_sent': current kdps being sent,\n 'total_bytes_sent': total bytes sent,\n 'total_bytes_received': total bytes received,\n 'queries_received': number of queries received per second,\n 'queries_sent': number of queries sent per second,\n 'recent_contacts': count of recently contacted peers,\n 'unique_contacts': count of unique peers\n },\n }",
"title": "status"
},
{
"location": "/#stream_availability",
"text": "Get stream availability for lbry uri\n\nArgs:\n 'uri' (required) : (str) check availability for this uri\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n 'is_available': bool ,\n 'did_decode': bool ,\n 'did_resolve': bool ,\n 'is_stream': bool ,\n 'num_blobs_in_stream': int ,\n 'sd_hash': str ,\n 'sd_blob_availability': dict see `blob_availability`,\n 'head_blob_hash': str ,\n 'head_blob_availability': dict see `blob_availability`,\n 'use_upnp': bool ,\n 'upnp_redirect_is_set': bool ,\n 'error': None | str error message\n }",
"text": "Get stream availability for lbry uri\n\nArgs:\n 'uri' (required) : (str) check availability for this uri\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'search_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n 'is_available': bool ,\n 'did_decode': bool ,\n 'did_resolve': bool ,\n 'is_stream': bool ,\n 'num_blobs_in_stream': int ,\n 'sd_hash': str ,\n 'sd_blob_availability': dict see `blob_availability`,\n 'head_blob_hash': str ,\n 'head_blob_availability': dict see `blob_availability`,\n 'use_upnp': bool ,\n 'upnp_redirect_is_set': bool ,\n 'error': None | str error message\n }",
"title": "stream_availability"
},
{
@@ -277,7 +262,7 @@
},
{
"location": "/#wallet_send",
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nArgs:\n 'amount' (required) : (Decimal) amount of credit to send\n 'address' (required) : (str) address to send credits to\n 'claim_id' (required) : (str) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nArgs:\n 'amount' (required) : (float) amount of credit to send\n 'address' (required) : (str) address to send credits to\n 'claim_id' (required) : (float) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
"title": "wallet_send"
},
{
@@ -292,7 +277,7 @@
},
{
"location": "/cli/",
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [--announce_all] [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --announce_all=\nannounce_all\n : (bool) announce all the blobs possessed by user\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing informatLine truncated
"title": "CLI"
},
{
@@ -302,7 +287,7 @@
},
{
"location": "/cli/#blob_announce",
"text": "Announce blobs to the DHT\n\nUsage:\n blob_announce [ blob_hash | --blob_hash= blob_hash ]\n [ stream_hash | --stream_hash= stream_hash ] | [ sd_hash | --sd_hash= sd_hash ]\n\n\nOptions:\n --blob_hash= blob_hash : (str) announce a blob, specified by blob_hash\n --stream_hash= stream_hash : (str) announce all blobs associated with\n stream_hash\n --sd_hash= sd_hash : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful",
"text": "Announce blobs to the DHT\n\nUsage:\n blob_announce [--announce_all] [ blob_hash | --blob_hash= blob_hash ]\n [ stream_hash | --stream_hash= stream_hash ]\n [ sd_hash | --sd_hash= sd_hash ]\n\n\nOptions:\n --announce_all= announce_all : (bool) announce all the blobs possessed by user\n --blob_hash= blob_hash : (str) announce a blob, specified by blob_hash\n --stream_hash= stream_hash : (str) announce all blobs associated with\n stream_hash\n --sd_hash= sd_hash : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful",
"title": "blob_announce"
},
{
@@ -325,11 +310,6 @@
"text": "Returns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [ uri | --uri= uri ]\n [ stream_hash | --stream_hash= stream_hash ]\n [ sd_hash | --sd_hash= sd_hash ]\n [ page_size | --page_size= page_size ]\n [ page | --page= page ]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri= uri : (str) filter blobs by stream in a uri\n --stream_hash= stream_hash : (str) filter blobs by stream hash\n --sd_hash= sd_hash : (str) filter blobs by sd hash\n --page_size= page_size : (int) results page size\n --page= page : (int) page of results to return\n\nReturns:\n (list) List of blob hashes",
"title": "blob_list"
},
{
"location": "/cli/#blob_reflect",
"text": "Reflects specified blobs\n\nUsage:\n blob_reflect ( blob_hashes ...) [--reflector_server= reflector_server ]\n\n\nOptions:\n --reflector_server= reflector_server : (str) reflector address\n\nReturns:\n (list) reflected blob hashes",
"title": "blob_reflect"
},
{
"location": "/cli/#blob_reflect_all",
"text": "Reflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful",
@@ -347,7 +327,7 @@
},
{
"location": "/cli/#channel_import",
"text": "Import serialized channel signing information (to allow signing new claims to the channel)\n\nUsage:\n channel_import ( serialized_certificate_info | --serialized_certificate_info= serialized_certificate_info )\n\n\nOptions:\n --serialized_certificate_info= serialized_certificate_info : (str) certificate info\n\nReturns:\n (dict) Result dictionary",
"text": "Import serialized channel signing information (to allow signing new claims to the channel)\n\nUsage:\n channel_import ( serialized_certificate_info |\n --serialized_certificate_info= serialized_certificate_info )\n\n\nOptions:\n --serialized_certificate_info= serialized_certificate_info : (str) certificate info\n\nReturns:\n (dict) Result dictionary",
"title": "channel_import"
},
{
@@ -377,7 +357,7 @@
},
{
"location": "/cli/#claim_list_mine",
"text": "List my name claims\n\nUsage:\n claim_list_mine\n\n\nOptions:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the claim\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
"text": "List my name claims\n\nUsage:\n claim_list_mine\n\n\nOptions:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the cliam\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
"title": "claim_list_mine"
},
{
@@ -402,7 +382,7 @@
},
{
"location": "/cli/#cli_test_command",
"text": "This command is only for testing the CLI argument parsing\nUsage:\n cli_test_command [--a_arg] [--b_arg] ( pos_arg | --pos_arg= pos_arg )\n [ pos_args ...] [--pos_arg2= pos_arg2 ]\n [--pos_arg3= pos_arg3 ]\n\n\nOptions:\n --a_arg : (bool) a arg\n --b_arg : (bool) b arg\n --pos_arg= pos_arg : (int) pos arg\n --pos_args= pos_args : (int) pos args\n --pos_arg2= pos_arg2 : (int) pos arg 2\n --pos_arg3= pos_arg3 : (int) pos arg 3\n\nReturns:\n pos args",
"text": "This command is only for testing the CLI argument parsing\nUsage:\n cli_test_command [--a_arg] [--b_arg] ( pos_arg | --pos_arg= pos_arg )\n [ pos_args ...] [--pos_arg2= pos_arg2 ]\n [--pos_arg3= pos_arg3 ]\n\n\nOptions:\n --a_arg : a arg\n --b_arg : b arg\n --pos_arg= pos_arg : pos arg\n --pos_args= pos_args : pos args\n --pos_arg2= pos_arg2 : pos arg 2\n --pos_arg3= pos_arg3 : pos arg 3\n\nReturns:\n pos args",
"title": "cli_test_command"
},
{
@@ -422,7 +402,7 @@
},
{
"location": "/cli/#file_list",
"text": "List files limited by optional filters\n\nUsage:\n file_list [--sd_hash= sd_hash ] [--file_name= file_name ] [--stream_hash= stream_hash ]\n [--rowid= rowid ] [--claim_id= claim_id ] [--outpoint= outpoint ] [--txid= txid ] [--nout= nout ]\n [--channel_claim_id= channel_claim_id ] [--channel_name= channel_name ]\n [--claim_name= claim_name ] [--full_status] [--sort= sort_method ...]\n\n\nOptions:\n --sd_hash= sd_hash : (str) get file with matching sd hash\n --file_name= file_name : (str) get file with matching file name in the\n downloads folder\n --stream_hash= stream_hash : (str) get file with matching stream hash\n --rowid= rowid : (int) get file with matching row id\n --claim_id= claim_id : (str) get file with matching claim id\n --outpoint= outpoint : (str) get file with matching claim outpoint\n --txid= txid : (str) get file with matching claim txid\n --nout= nout : (int) get file with matching claim nout\n --channel_claim_id= channel_claim_id : (str) get file with matching channel claim id\n --channel_name= channel_name : (str) get file with matching channel name\n --claim_name= claim_name : (str) get file with matching claim name\n --full_status : (bool) full status, populate the\n 'message' and 'size' fields\n --sort= sort_method : (str) sort by any property, like 'file_name'\n or 'metadata.author'; to specify direction\n append ',asc' or ',desc'\n\nReturns:\n (list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes, None if full_status is false,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) num_completed, None if full_status is false,\n 'blobs_in_stream': (int) None if full_status is false,\n 'status': (str) downloader status, None if full_status is false,\n 'claim_id': (str) None if full_status is false or if claim is not found,\n 'outpoint': (str) None if full_status is false or if claim is not found,\n 'txid': (str) None if full_status is false or if claim is not found,\n 'nout': (int) None if full_status is false or if claim is not found,\n 'metadata': (dict) None if full_status is false or if claim is not found,\n 'channel_claim_id': (str) None if full_status is false or if claim is not found or signed,\n 'channel_name': (str) None if full_status is false or if claim is not found or signed,\n 'claim_name': (str) None if full_status is false or if claim is not found\n },\n ]",
"text": "List files limited by optional filters\n\nUsage:\n file_list [--sd_hash= sd_hash ] [--file_name= file_name ] [--stream_hash= stream_hash ]\n [--rowid= rowid ] [--claim_id= claim_id ] [--outpoint= outpoint ] [--txid= txid ] [--nout= nout ]\n [--channel_claim_id= channel_claim_id ] [--channel_name= channel_name ]\n [--claim_name= claim_name ] [--full_status]\n\n\nOptions:\n --sd_hash= sd_hash : (str) get file with matching sd hash\n --file_name= file_name : (str) get file with matching file name in the\n downloads folder\n --stream_hash= stream_hash : (str) get file with matching stream hash\n --rowid= rowid : (int) get file with matching row id\n --claim_id= claim_id : (str) get file with matching claim id\n --outpoint= outpoint : (str) get file with matching claim outpoint\n --txid= txid : (str) get file with matching claim txid\n --nout= nout : (int) get file with matching claim nout\n --channel_claim_id= channel_claim_id : (str) get file with matching channel claim id\n --channel_name= channel_name : (str) get file with matching channel name\n --claim_name= claim_name : (str) get file with matching claim name\n --full_status : (bool) full status, populate the\n 'message' and 'size' fields\n\nReturns:\n (list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes, None if full_status is false,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) num_completed, None if full_status is false,\n 'blobs_in_stream': (int) None if full_status is false,\n 'status': (str) downloader status, None if full_status is false,\n 'claim_id': (str) None if full_status is false or if claim is not found,\n 'outpoint': (str) None if full_status is false or if claim is not found,\n 'txid': (str) None if full_status is false or if claim is not found,\n 'nout': (int) None if full_status is false or if claim is not found,\n 'metadata': (dict) None if full_status is false or if claim is not found,\n 'channel_claim_id': (str) None if full_status is false or if claim is not found or signed,\n 'channel_name': (str) None if full_status is false or if claim is not found or signed,\n 'claim_name': (str) None if full_status is false or if claim is not found\n },\n ]",
"title": "file_list"
},
{
@@ -447,24 +427,14 @@
},
{
"location": "/cli/#peer_list",
"text": "Get peers for blob hash\n\nUsage:\n peer_list ( blob_hash | --blob_hash= blob_hash ) [ timeout | --timeout= timeout ]\n\n\nOptions:\n --blob_hash= blob_hash : (str) find available peers for this blob hash\n --timeout= timeout : (int) peer search timeout in seconds\n\nReturns:\n (list) List of contact dictionaries {'host': peer ip , 'port': peer port , 'node_id': peer node id }",
"text": "Get peers for blob hash\n\nUsage:\n peer_list ( blob_hash | --blob_hash= blob_hash ) [ timeout | --timeout= timeout ]\n\n\nOptions:\n --blob_hash= blob_hash : (str) find available peers for this blob hash\n --timeout= timeout : (int) peer search timeout in seconds\n\nReturns:\n (list) List of contacts",
"title": "peer_list"
},
{
"location": "/cli/#peer_ping",
"text": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.\n\nUsage:\n peer_ping ( node_id | --node_id= node_id ) [ address | --address= address ] [ port | --port= port ]\n\n\nOptions:\n --address= address : (str) ip address of the peer\n --port= port : (int) udp port of the peer\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
"title": "peer_ping"
},
{
"location": "/cli/#publish",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (float) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': float,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) title of the publication\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"title": "publish"
},
{
"location": "/cli/#report_bug",
"text": "Report a bug to slack\n\nUsage:\n report_bug ( message | --message= message )\n\n\nOptions:\n --message= message : (str) Description of the bug\n\nReturns:\n (bool) true if successful",
"title": "report_bug"
},
{
"location": "/cli/#resolve",
"text": "Resolve given LBRY URIs\n\nUsage:\n resolve [--force] ( uri | --uri= uri ) [ uris ...]\n\n\nOptions:\n --force : (bool) force refresh and ignore cache\n --uri= uri : (str) uri to resolve\n --uris= uris : (list) uris to resolve\n\nReturns:\n Dictionary of results, keyed by uri\n ' uri ': {\n If a resolution error occurs:\n 'error': Error message\n\n If the uri resolves to a channel or a claim in a channel:\n 'certificate': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the certificate claim,\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}],\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n\n If the uri resolves to a channel:\n 'claims_in_channel': (int) number of claims in the channel,\n\n If the uri resolves to a claim:\n 'claim': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the claim,\n 'channel_name': (str) channel name if claim is in a channel\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}]\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n }",
@@ -477,7 +447,7 @@
},
{
"location": "/cli/#routing_table_get",
"text": "Get DHT routing information\n\nUsage:\n routing_table_get\n\n\nOptions:\n None\n\nReturns:\n (dict) dictionary containing routing and contact information\n {\n buckets : {\n bucket index : [\n {\n address : (str) peer address,\n port : (int) peer udp port\n node_id : (str) peer node id,\n blobs : (list) blob hashes announced by peer\n }\n ]\n },\n contacts : (list) contact node ids,\n blob_hashes : (list) all of the blob hashes stored by peers in the list of buckets,\n node_id : (str) the local dht node id\n }",
"text": "Get DHT routing information\n\nUsage:\n routing_table_get\n\n\nOptions:\n None\n\nReturns:\n (dict) dictionary containing routing and contact information\n {\n buckets : {\n bucket index : [\n {\n address : (str) peer address,\n node_id : (str) peer node id,\n blobs : (list) blob hashes announced by peer\n }\n ]\n },\n contacts : (list) contact node ids,\n blob_hashes : (list) all of the blob hashes stored by peers in the list of buckets,\n node_id : (str) the local dht node id\n }",
"title": "routing_table_get"
},
{
@@ -492,12 +462,12 @@
},
{
"location": "/cli/#status",
"text": "Get daemon status\n\nUsage:\n status\n\n\nOptions:\n None\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }",
"text": "Get daemon status\n\nUsage:\n status [--session_status] [--dht_status]\n\n\nOptions:\n --session_status : (bool) include session status in results\n --dht_status : (bool) include dht network and peer status\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'lbry_id': lbry peer id, base58,\n 'installation_id': installation id, base58,\n 'is_running': bool,\n 'is_first_run': bool,\n 'startup_status': {\n 'code': status code,\n 'message': status message\n },\n 'connection_status': {\n 'code': connection status code,\n 'message': connection status message\n },\n 'blockchain_status': {\n 'blocks': local blockchain height,\n 'blocks_behind': remote_height - local_height,\n 'best_blockhash': block hash of most recent block,\n },\n 'wallet_is_encrypted': bool,\n\n If given the session status option:\n 'session_status': {\n 'managed_blobs': count of blobs in the blob manager,\n 'managed_streams': count of streams in the file manager\n 'announce_queue_size': number of blobs currently queued to be announced\n 'should_announce_blobs': number of blobs that should be announced\n }\n\n If given the dht status option:\n 'dht_status': {\n 'kbps_received': current kbps receiving,\n 'kbps_sent': current kdps being sent,\n 'total_bytes_sent': total bytes sent,\n 'total_bytes_received': total bytes received,\n 'queries_received': number of queries received per second,\n 'queries_sent': number of queries sent per second,\n 'recent_contacts': count of recently contacted peers,\n 'unique_contacts': count of unique peers\n },\n }",
"title": "status"
},
{
"location": "/cli/#stream_availability",
"text": "Get stream availability for lbry uri\n\nUsage:\n stream_availability ( uri | --uri= uri )\n [ search_timeout | --search_timeout= search_timeout ]\n [ blob_timeout | --blob_timeout= blob_timeout ]\n\n\nOptions:\n --uri= uri : (str) check availability for this uri\n --search_timeout= search_timeout : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout= blob_timeout : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n 'is_available': bool ,\n 'did_decode': bool ,\n 'did_resolve': bool ,\n 'is_stream': bool ,\n 'num_blobs_in_stream': int ,\n 'sd_hash': str ,\n 'sd_blob_availability': dict see `blob_availability`,\n 'head_blob_hash': str ,\n 'head_blob_availability': dict see `blob_availability`,\n 'use_upnp': bool ,\n 'upnp_redirect_is_set': bool ,\n 'error': None | str error message\n }",
"text": "Get stream availability for lbry uri\n\nUsage:\n stream_availability ( uri | --uri= uri )\n [ search_timeout | --search_timeout= search_timeout ]\n [ blob_timeout | --blob_timeout= blob_timeout ]\n\n\nOptions:\n --uri= uri : (str) check availability for this uri\n --search_timeout= search_timeout : (int) how long to search for peers for the blob\n in the dht\n --search_timeout= blob_timeout : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n 'is_available': bool ,\n 'did_decode': bool ,\n 'did_resolve': bool ,\n 'is_stream': bool ,\n 'num_blobs_in_stream': int ,\n 'sd_hash': str ,\n 'sd_blob_availability': dict see `blob_availability`,\n 'head_blob_hash': str ,\n 'head_blob_availability': dict see `blob_availability`,\n 'use_upnp': bool ,\n 'upnp_redirect_is_set': bool ,\n 'error': None | str error message\n }",
"title": "stream_availability"
},
{
@@ -567,7 +537,7 @@
},
{
"location": "/cli/#wallet_send",
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nUsage:\n wallet_send ( amount | --amount= amount )\n (( address | --address= address ) | ( claim_id | --claim_id= claim_id ))\n\n\nOptions:\n --amount= amount : (Decimal) amount of credit to send\n --address= address : (str) address to send credits to\n --claim_id= claim_id : (str) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nUsage:\n wallet_send ( amount | --amount= amount )\n (( address | --address= address ) | ( claim_id | --claim_id= claim_id ))\n\n\nOptions:\n --amount= amount : (float) amount of credit to send\n --address= address : (str) address to send credits to\n --claim_id= claim_id : (float) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
"title": "wallet_send"
},
{
-23
View File
@@ -1,23 +0,0 @@
# This is an example daemon_settings.yml file.
# See https://lbry.tech/resources/daemon-settings for all configuration keys and values
share_usage_data: True
lbryum_servers:
- lbryumx1.lbry.io:50001
- lbryumx2.lbry.io:50001
blockchain_name: lbrycrd_main
data_dir: /home/lbry/.lbrynet
download_directory: /home/lbry/downloads
delete_blobs_on_remove: True
dht_node_port: 4444
peer_port: 3333
use_upnp: True
#components_to_skip:
# - peer_protocol_server
# - hash_announcer
# - dht
BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.6 KiB

+4 -2
View File
@@ -1,4 +1,6 @@
__name__ = "lbrynet"
__version__ = "0.31.0rc3"
import logging
__version__ = "0.19.3rc3"
version = tuple(__version__.split('.'))
logging.getLogger(__name__).addHandler(logging.NullHandler())
+268
View File
@@ -0,0 +1,268 @@
import collections
import logging
from twisted.internet import defer, task
from requests import auth
from txrequests import Session
from lbrynet import conf
from lbrynet.core import looping_call_manager, utils, system_info
# Things We Track
SERVER_STARTUP = 'Server Startup'
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
SERVER_STARTUP_ERROR = 'Server Startup Error'
DOWNLOAD_STARTED = 'Download Started'
DOWNLOAD_ERRORED = 'Download Errored'
DOWNLOAD_FINISHED = 'Download Finished'
HEARTBEAT = 'Heartbeat'
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
NEW_CHANNEL = 'New Channel'
CREDITS_SENT = 'Credits Sent'
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
log = logging.getLogger(__name__)
class Manager(object):
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
self.analytics_api = analytics_api
self._tracked_data = collections.defaultdict(list)
self.looping_call_manager = self._setup_looping_calls()
self.context = context or self._make_context(
system_info.get_platform(), conf.settings['wallet'])
self.installation_id = installation_id or conf.settings.installation_id
self.session_id = session_id or conf.settings.get_session_id()
self.is_started = False
@classmethod
def new_instance(cls, enabled=None):
api = Api.new_instance(enabled)
return cls(api)
# Things We Track
def send_server_startup(self):
self.analytics_api.track(self._event(SERVER_STARTUP))
def send_server_startup_success(self):
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
def send_server_startup_error(self, message):
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
def send_download_started(self, id_, name, claim_dict=None):
self.analytics_api.track(
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
)
def send_download_errored(self, err, id_, name, claim_dict, report):
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
report)
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
def send_download_finished(self, id_, name, report, claim_dict=None):
download_properties = self._download_properties(id_, name, claim_dict, report)
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
def send_claim_action(self, action):
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
def send_new_channel(self):
self.analytics_api.track(self._event(NEW_CHANNEL))
def send_credits_sent(self):
self.analytics_api.track(self._event(CREDITS_SENT))
def _send_heartbeat(self):
self.analytics_api.track(self._event(HEARTBEAT))
def _update_tracked_metrics(self):
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
if should_send:
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
# Setup / Shutdown
def start(self):
if not self.is_started:
for name, _, interval in self._get_looping_calls():
self.looping_call_manager.start(name, interval)
self.is_started = True
def shutdown(self):
self.looping_call_manager.shutdown()
def register_repeating_metric(self, event_name, value_generator, frequency=300):
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
self.looping_call_manager.register_looping_call(event_name, lcall)
lcall.start(frequency)
def _get_looping_calls(self):
return [
('send_heartbeat', self._send_heartbeat, 60),
('update_tracked_metrics', self._update_tracked_metrics, 300),
]
def _setup_looping_calls(self):
call_manager = looping_call_manager.LoopingCallManager()
for name, fn, _ in self._get_looping_calls():
call_manager.register_looping_call(name, task.LoopingCall(fn))
return call_manager
def _send_repeating_metric(self, event_name, value_generator):
result = value_generator()
self._if_deferred(result, self._send_repeating_metric_value, event_name)
def _send_repeating_metric_value(self, result, event_name):
should_send, value = result
if should_send:
self.analytics_api.track(self._metric_event(event_name, value))
def add_observation(self, metric, value):
self._tracked_data[metric].append(value)
def summarize_and_reset(self, metric, op=sum):
"""Apply `op` on the current values for `metric`.
This operation also resets the metric.
Returns:
a tuple (should_send, value)
"""
try:
values = self._tracked_data.pop(metric)
return True, op(values)
except KeyError:
return False, None
def _event(self, event, event_properties=None):
return {
'userId': 'lbry',
'event': event,
'properties': self._event_properties(event_properties),
'context': self.context,
'timestamp': utils.isonow()
}
def _metric_event(self, metric_name, value):
return self._event(metric_name, {'value': value})
def _event_properties(self, event_properties=None):
properties = {
'lbry_id': self.installation_id,
'session_id': self.session_id,
}
properties.update(event_properties or {})
return properties
@staticmethod
def _download_properties(id_, name, claim_dict=None, report=None):
sd_hash = None if not claim_dict else claim_dict.source_hash
p = {
'download_id': id_,
'name': name,
'stream_info': sd_hash
}
if report:
p['report'] = report
return p
@staticmethod
def _download_error_properties(error, id_, name, claim_dict, report):
def error_name(err):
if not hasattr(type(err), "__name__"):
return str(type(err))
return type(err).__name__
return {
'download_id': id_,
'name': name,
'stream_info': claim_dict.source_hash,
'error': error_name(error),
'reason': error.message,
'report': report
}
@staticmethod
def _make_context(platform, wallet):
return {
'app': {
'name': 'lbrynet',
'version': platform['lbrynet_version'],
'python_version': platform['python_version'],
'build': platform['build'],
'wallet': {
'name': wallet,
'version': platform['lbryum_version'] if wallet == conf.LBRYUM_WALLET else None
},
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
'library': {
'name': 'lbrynet-analytics',
'version': '1.0.0'
},
}
@staticmethod
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
if isinstance(maybe_deferred, defer.Deferred):
maybe_deferred.addCallback(callback, *args, **kwargs)
else:
callback(maybe_deferred, *args, **kwargs)
class Api(object):
def __init__(self, session, url, write_key, enabled):
self.session = session
self.url = url
self._write_key = write_key
self._enabled = enabled
def _post(self, endpoint, data):
# there is an issue with a timing condition with keep-alive
# that is best explained here: https://github.com/mikem23/keepalive-race
#
# If you make a request, wait just the right amount of time,
# then make another request, the requests module may opt to
# reuse the connection, but by the time the server gets it the
# timeout will have expired.
#
# by forcing the connection to close, we will disable the keep-alive.
assert endpoint[0] == '/'
headers = {"Connection": "close"}
return self.session.post(
self.url + endpoint,
json=data,
auth=auth.HTTPBasicAuth(self._write_key, ''),
headers=headers
)
def track(self, event):
"""Send a single tracking event"""
if not self._enabled:
return defer.succeed('Analytics disabled')
def _log_error(failure, event):
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
log.debug('Sending track event: %s', event)
d = self._post('/track', event)
d.addErrback(_log_error, event)
return d
@classmethod
def new_instance(cls, enabled=None):
"""Initialize an instance using values from the configuration"""
session = Session()
if enabled is None:
enabled = conf.settings['share_usage_data']
return cls(
session,
conf.settings['ANALYTICS_ENDPOINT'],
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
enabled,
)
+1
View File
@@ -0,0 +1 @@
import paths
+26
View File
@@ -0,0 +1,26 @@
# App root files dir
# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29
def android_files_dir():
return None
# Base internal storage path
def android_internal_storage_dir():
return None
# Base external (SD card, if present) storage path
def android_external_storage_dir():
return None
# Internal device storage (private app folder)
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
def android_app_internal_storage_dir():
return None
# External (app folder on SD card, if present) storage
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
def android_app_external_storage_dir():
return None
+4 -6
View File
@@ -1,6 +1,4 @@
from lbrynet.cryptoutils import get_lbry_hash_obj
MAX_BLOB_SIZE = 2 * 2 ** 20
# digest_size is in bytes, and blob hashes are hex encoded
blobhash_length = get_lbry_hash_obj().digest_size * 2
from blob_file import BlobFile
from creator import BlobFileCreator
from writer import HashBlobWriter
from reader import HashBlobReader
+166 -144
View File
@@ -1,48 +1,19 @@
import os
import asyncio
import binascii
import logging
import typing
from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
from lbrynet.cryptoutils import backend, get_lbry_hash_obj
from lbrynet.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
from lbrynet.blob import MAX_BLOB_SIZE, blobhash_length
from lbrynet.blob.blob_info import BlobInfo
import os
from twisted.internet import defer, threads
from twisted.web.client import FileBodyProducer
from twisted.python.failure import Failure
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
from lbrynet.core.utils import is_valid_blobhash
from lbrynet.blob.writer import HashBlobWriter
from lbrynet.blob.reader import HashBlobReader
log = logging.getLogger(__name__)
def is_valid_hashcharacter(char: str) -> bool:
return char in "0123456789abcdef"
MAX_BLOB_SIZE = 2 * 2 ** 20
def is_valid_blobhash(blobhash: str) -> bool:
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
@param blobhash: string, the blobhash to check
@return: True/False
"""
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
padder = PKCS7(AES.block_size).padder()
encryptor = cipher.encryptor()
encrypted = encryptor.update(padder.update(unencrypted) + padder.finalize()) + encryptor.finalize()
digest = get_lbry_hash_obj()
digest.update(encrypted)
return encrypted, digest.hexdigest()
class BlobFile:
class BlobFile(object):
"""
A chunk of data available on the network which is specified by a hashsum
@@ -51,137 +22,188 @@ class BlobFile:
Also can be used for reading from blobs on the local filesystem
"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, blob_hash: str,
length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['BlobFile'], typing.Awaitable]] = None):
def __str__(self):
return self.blob_hash[:16]
def __repr__(self):
return '<{}({})>'.format(self.__class__.__name__, str(self))
def __init__(self, blob_dir, blob_hash, length=None):
if not is_valid_blobhash(blob_hash):
raise InvalidBlobHashError(blob_hash)
self.loop = loop
self.blob_hash = blob_hash
self.length = length
self.writers = {} # {Peer: writer, finished_deferred}
self._verified = False
self.readers = 0
self.blob_dir = blob_dir
self.file_path = os.path.join(blob_dir, self.blob_hash)
self.writers: typing.List[HashBlobWriter] = []
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
self.finished_writing = asyncio.Event(loop=loop)
self.blob_write_lock = asyncio.Lock(loop=loop)
if os.path.isfile(os.path.join(blob_dir, blob_hash)):
length = length or int(os.stat(os.path.join(blob_dir, blob_hash)).st_size)
self.length = length
self.verified.set()
self.finished_writing.set()
self.blob_write_lock = defer.DeferredLock()
self.saved_verified_blob = False
self.blob_completed_callback = blob_completed_callback
if os.path.isfile(self.file_path):
self.set_length(os.path.getsize(self.file_path))
# This assumes that the hash of the blob has already been
# checked as part of the blob creation process. It might
# be worth having a function that checks the actual hash;
# its probably too expensive to have that check be part of
# this call.
self._verified = True
def writer_finished(self, writer: HashBlobWriter):
def callback(finished: asyncio.Future):
try:
error = finished.result()
except Exception as err:
error = err
if writer in self.writers: # remove this download attempt
self.writers.remove(writer)
if not error: # the blob downloaded, cancel all the other download attempts and set the result
while self.writers:
other = self.writers.pop()
other.finished.cancel()
t = self.loop.create_task(self.save_verified_blob(writer))
t.add_done_callback(lambda *_: self.finished_writing.set())
return
if isinstance(error, (InvalidBlobHashError, InvalidDataError)):
log.error("writer error downloading %s: %s", self.blob_hash[:8], str(error))
elif not isinstance(error, (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError)):
log.exception("something else")
raise error
return callback
async def save_verified_blob(self, writer):
def _save_verified():
# log.debug(f"write blob file {self.blob_hash[:8]} from {writer.peer.address}")
if not self.saved_verified_blob and not os.path.isfile(self.file_path):
if self.get_length() == len(writer.verified_bytes):
with open(self.file_path, 'wb') as write_handle:
write_handle.write(writer.verified_bytes)
self.saved_verified_blob = True
else:
raise Exception("length mismatch")
if self.verified.is_set():
return
async with self.blob_write_lock:
await self.loop.run_in_executor(None, _save_verified)
if self.blob_completed_callback:
await self.blob_completed_callback(self)
self.verified.set()
def open_for_writing(self) -> HashBlobWriter:
if os.path.exists(self.file_path):
raise OSError(f"File already exists '{self.file_path}'")
fut = asyncio.Future(loop=self.loop)
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
self.writers.append(writer)
fut.add_done_callback(self.writer_finished(writer))
return writer
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
def open_for_writing(self, peer):
"""
Read and send the file to the writer and return the number of bytes sent
open a blob file to be written by peer, supports concurrent
writers, as long as they are from differnt peers.
returns tuple of (writer, finished_deferred)
writer - a file like object with a write() function, close() when finished
finished_deferred - deferred that is fired when write is finished and returns
a instance of itself as HashBlob
"""
if not peer in self.writers:
log.debug("Opening %s to be written by %s", str(self), str(peer))
finished_deferred = defer.Deferred()
writer = HashBlobWriter(self.get_length, self.writer_finished)
self.writers[peer] = (writer, finished_deferred)
return (writer, finished_deferred)
log.warning("Tried to download the same file twice simultaneously from the same peer")
return None, None
with open(self.file_path, 'rb') as handle:
return await self.loop.sendfile(writer.transport, handle)
def open_for_reading(self):
"""
open blob for reading
async def close(self):
while self.writers:
self.writers.pop().finished.cancel()
returns a file like object that can be read() from, and closed() when
finished
"""
if self._verified is True:
f = open(self.file_path, 'rb')
reader = HashBlobReader(f, self.reader_finished)
self.readers += 1
return reader
return None
async def delete(self):
await self.close()
async with self.blob_write_lock:
def delete(self):
"""
delete blob file from file system, prevent deletion
if a blob is being read from or written to
returns a deferred that firesback when delete is completed
"""
if not self.writers and not self.readers:
self._verified = False
self.saved_verified_blob = False
if os.path.isfile(self.file_path):
os.remove(self.file_path)
def decrypt(self, key: bytes, iv: bytes) -> bytes:
"""
Decrypt a BlobFile to plaintext bytes
"""
def delete_from_file_system():
if os.path.isfile(self.file_path):
os.remove(self.file_path)
with open(self.file_path, "rb") as f:
buff = f.read()
if len(buff) != self.length:
raise ValueError("unexpected length")
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
unpadder = PKCS7(AES.block_size).unpadder()
decryptor = cipher.decryptor()
return unpadder.update(decryptor.update(buff) + decryptor.finalize()) + unpadder.finalize()
d = threads.deferToThread(delete_from_file_system)
@classmethod
async def create_from_unencrypted(cls, loop: asyncio.BaseEventLoop, blob_dir: str, key: bytes,
iv: bytes, unencrypted: bytes, blob_num: int) -> BlobInfo:
"""
Create an encrypted BlobFile from plaintext bytes
"""
def log_error(err):
log.warning("An error occurred deleting %s: %s",
str(self.file_path), err.getErrorMessage())
return err
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
length = len(blob_bytes)
blob = cls(loop, blob_dir, blob_hash, length)
writer = blob.open_for_writing()
writer.write(blob_bytes)
await blob.verified.wait()
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
d.addErrback(log_error)
return d
else:
return defer.fail(Failure(
ValueError("File is currently being read or written and cannot be deleted")))
@property
def verified(self):
"""
Protect verified from being modified by other classes.
verified is True if a write to a blob has completed succesfully,
or a blob has been read to have the same length as specified
in init
"""
return self._verified
def set_length(self, length):
if self.length is not None and length == self.length:
return
return True
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
self.length = length
return
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
return True
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s",
self.length, length)
return False
def get_length(self):
return self.length
def get_is_verified(self):
return self.verified.is_set()
return self.verified
def is_downloading(self):
if self.writers:
return True
return False
def reader_finished(self, reader):
self.readers -= 1
return defer.succeed(True)
def writer_finished(self, writer, err=None):
def fire_finished_deferred():
self._verified = True
for p, (w, finished_deferred) in self.writers.items():
if w == writer:
del self.writers[p]
finished_deferred.callback(self)
return True
log.warning(
"Somehow, the writer that was accepted as being valid was already removed: %s",
writer)
return False
def errback_finished_deferred(err):
for p, (w, finished_deferred) in self.writers.items():
if w == writer:
del self.writers[p]
finished_deferred.errback(err)
def cancel_other_downloads():
for p, (w, finished_deferred) in self.writers.items():
w.close()
if err is None:
if writer.len_so_far == self.length and writer.blob_hash == self.blob_hash:
if self._verified is False:
d = self.save_verified_blob(writer)
d.addCallbacks(lambda _: fire_finished_deferred(), errback_finished_deferred)
d.addCallback(lambda _: cancel_other_downloads())
else:
d = defer.succeed(None)
fire_finished_deferred()
else:
if writer.len_so_far != self.length:
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
else:
err_string = "blob hash is %s vs expected %s" % (writer.blob_hash, self.blob_hash)
errback_finished_deferred(Failure(InvalidDataError(err_string)))
d = defer.succeed(None)
else:
errback_finished_deferred(err)
d = defer.succeed(None)
d.addBoth(lambda _: writer.close_handle())
return d
def save_verified_blob(self, writer):
# we cannot have multiple _save_verified_blob interrupting
# each other, can happen since startProducing is a deferred
return self.blob_write_lock.run(self._save_verified_blob, writer)
@defer.inlineCallbacks
def _save_verified_blob(self, writer):
if self.saved_verified_blob is False:
writer.write_handle.seek(0)
out_path = os.path.join(self.blob_dir, self.blob_hash)
producer = FileBodyProducer(writer.write_handle)
yield producer.startProducing(open(out_path, 'wb'))
self.saved_verified_blob = True
defer.returnValue(True)
else:
raise DownloadCanceledError()
-19
View File
@@ -1,19 +0,0 @@
import typing
class BlobInfo:
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
self.blob_hash = blob_hash
self.blob_num = blob_num
self.length = length
self.iv = iv
def as_dict(self) -> typing.Dict:
d = {
'length': self.length,
'blob_num': self.blob_num,
'iv': self.iv,
}
if self.blob_hash: # non-terminator blobs have a blob hash
d['blob_hash'] = self.blob_hash
return d
-89
View File
@@ -1,89 +0,0 @@
import typing
import asyncio
import logging
from sqlite3 import IntegrityError
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.blob.blob_file import BlobFile
from lbrynet.stream.descriptor import StreamDescriptor
if typing.TYPE_CHECKING:
from lbrynet.dht.protocol.data_store import DictDataStore
log = logging.getLogger(__name__)
class BlobFileManager:
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: SQLiteStorage,
node_data_store: typing.Optional['DictDataStore'] = None):
"""
This class stores blobs on the hard disk
blob_dir - directory where blobs are stored
storage - SQLiteStorage object
"""
self.loop = loop
self.blob_dir = blob_dir
self.storage = storage
self._node_data_store = node_data_store
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
else self._node_data_store.completed_blobs
self.blobs: typing.Dict[str, BlobFile] = {}
async def setup(self) -> bool:
raw_blob_hashes = await self.get_all_verified_blobs()
self.completed_blob_hashes.update(raw_blob_hashes)
return True
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
if blob_hash in self.blobs:
if length and self.blobs[blob_hash].length is None:
self.blobs[blob_hash].set_length(length)
else:
self.blobs[blob_hash] = BlobFile(self.loop, self.blob_dir, blob_hash, length, self.blob_completed)
return self.blobs[blob_hash]
def get_stream_descriptor(self, sd_hash):
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
async def blob_completed(self, blob: BlobFile):
if blob.blob_hash is None:
raise Exception("Blob hash is None")
if not blob.length:
raise Exception("Blob has a length of 0")
if blob.blob_hash not in self.completed_blob_hashes:
self.completed_blob_hashes.add(blob.blob_hash)
await self.storage.add_completed_blob(blob.blob_hash)
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
"""Returns of the blobhashes_to_check, which are valid"""
blobs = [self.get_blob(b) for b in blob_hashes]
return [blob.blob_hash for blob in blobs if blob.get_is_verified()]
async def set_should_announce(self, blob_hash: str, should_announce: bool):
now = self.loop.time()
return await self.storage.set_should_announce(blob_hash, now, should_announce)
async def get_all_verified_blobs(self) -> typing.List[str]:
blob_hashes = await self.storage.get_all_blob_hashes()
return self.check_completed_blobs(blob_hashes)
async def delete_blobs(self, blob_hashes: typing.List[str]):
bh_to_delete_from_db = []
for blob_hash in blob_hashes:
if not blob_hash:
continue
try:
blob = self.get_blob(blob_hash)
await blob.delete()
bh_to_delete_from_db.append(blob_hash)
except Exception as e:
log.warning("Failed to delete blob file. Reason: %s", e)
if blob_hash in self.completed_blob_hashes:
self.completed_blob_hashes.remove(blob_hash)
if blob_hash in self.blobs:
del self.blobs[blob_hash]
try:
await self.storage.delete_blobs_from_db(bh_to_delete_from_db)
except IntegrityError as err:
if str(err) != "FOREIGN KEY constraint failed":
raise err
+51
View File
@@ -0,0 +1,51 @@
import os
import logging
from io import BytesIO
from twisted.internet import defer
from twisted.web.client import FileBodyProducer
from lbrynet.core.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
class BlobFileCreator(object):
"""
This class is used to create blobs on the local filesystem
when we do not know the blob hash beforehand (i.e, when creating
a new stream)
"""
def __init__(self, blob_dir):
self.blob_dir = blob_dir
self.buffer = BytesIO()
self._is_open = True
self._hashsum = get_lbry_hash_obj()
self.len_so_far = 0
self.blob_hash = None
self.length = None
@defer.inlineCallbacks
def close(self):
self.length = self.len_so_far
self.blob_hash = self._hashsum.hexdigest()
if self.blob_hash and self._is_open and self.length > 0:
# do not save 0 length files (empty tail blob in streams)
# or if its been closed already
self.buffer.seek(0)
out_path = os.path.join(self.blob_dir, self.blob_hash)
producer = FileBodyProducer(self.buffer)
yield producer.startProducing(open(out_path, 'wb'))
self._is_open = False
if self.length > 0:
defer.returnValue(self.blob_hash)
else:
# 0 length files (empty tail blob in streams )
# must return None as their blob_hash for
# it to be saved properly by EncryptedFileMetadataManagers
defer.returnValue(None)
def write(self, data):
if not self._is_open:
raise IOError
self._hashsum.update(data)
self.len_so_far += len(data)
self.buffer.write(data)
+32
View File
@@ -0,0 +1,32 @@
import logging
log = logging.getLogger(__name__)
class HashBlobReader(object):
"""
This is a file like reader class that supports
read(size) and close()
"""
def __init__(self, read_handle, finished_cb):
self.finished_cb = finished_cb
self.finished_cb_d = None
self.read_handle = read_handle
def __del__(self):
if self.finished_cb_d is None:
log.warn("Garbage collection was called, but reader for %s was not closed yet",
self.read_handle.name)
self.close()
def read(self, size=-1):
return self.read_handle.read(size)
def close(self):
# if we've already closed and called finished_cb, do nothing
if self.finished_cb_d is not None:
return
self.read_handle.close()
self.finished_cb_d = self.finished_cb(self)
+39 -52
View File
@@ -1,71 +1,58 @@
import typing
import logging
import asyncio
from io import BytesIO
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from lbrynet.cryptoutils import get_lbry_hash_obj
from twisted.python.failure import Failure
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
from lbrynet.core.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
class HashBlobWriter:
def __init__(self, expected_blob_hash: str, get_length: typing.Callable[[], int],
finished: asyncio.Future):
self.expected_blob_hash = expected_blob_hash
self.get_length = get_length
self.buffer = BytesIO()
self.finished = finished
self.finished.add_done_callback(lambda *_: self.close_handle())
class HashBlobWriter(object):
def __init__(self, length_getter, finished_cb):
self.write_handle = BytesIO()
self.length_getter = length_getter
self.finished_cb = finished_cb
self.finished_cb_d = None
self._hashsum = get_lbry_hash_obj()
self.len_so_far = 0
self.verified_bytes = b''
def __del__(self):
if self.buffer is not None:
log.warning("Garbage collection was called, but writer was not closed yet")
self.close_handle()
if self.finished_cb_d is None:
log.warn("Garbage collection was called, but writer was not closed yet")
self.close()
def calculate_blob_hash(self) -> str:
@property
def blob_hash(self):
return self._hashsum.hexdigest()
def closed(self):
return self.buffer is None or self.buffer.closed
def write(self, data: bytes):
expected_length = self.get_length()
if not expected_length:
raise IOError("unknown blob length")
if self.buffer is None:
log.warning("writer has already been closed")
if not self.finished.done():
self.finished.cancel()
return
def write(self, data):
if self.write_handle is None:
log.exception("writer has already been closed")
raise IOError('I/O operation on closed file')
self._hashsum.update(data)
self.len_so_far += len(data)
if self.len_so_far > expected_length:
self.close_handle()
self.finished.set_result(InvalidDataError(
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
))
return
self.buffer.write(data)
if self.len_so_far == expected_length:
blob_hash = self.calculate_blob_hash()
if blob_hash != self.expected_blob_hash:
self.close_handle()
self.finished.set_result(InvalidBlobHashError(
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
))
return
self.buffer.seek(0)
self.verified_bytes = self.buffer.read()
self.close_handle()
if self.finished and not (self.finished.done() or self.finished.cancelled()):
self.finished.set_result(None)
if self.len_so_far > self.length_getter():
self.finished_cb_d = self.finished_cb(
self,
Failure(InvalidDataError("Length so far is greater than the expected length."
" %s to %s" % (self.len_so_far,
self.length_getter()))))
else:
self.write_handle.write(data)
if self.len_so_far == self.length_getter():
self.finished_cb_d = self.finished_cb(self)
def close_handle(self):
if self.buffer is not None:
self.buffer.close()
self.buffer = None
if self.write_handle is not None:
self.write_handle.close()
self.write_handle = None
def close(self, reason=None):
# if we've already called finished_cb because we either finished writing
# or closed already, do nothing
if self.finished_cb_d is not None:
return
if reason is None:
reason = Failure(DownloadCanceledError())
self.finished_cb_d = self.finished_cb(self, reason)
-168
View File
@@ -1,168 +0,0 @@
import asyncio
import logging
import typing
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.writer import HashBlobWriter
log = logging.getLogger(__name__)
class BlobExchangeClientProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10):
self.loop = loop
self.peer_port: typing.Optional[int] = None
self.peer_address: typing.Optional[str] = None
self.peer_timeout = peer_timeout
self.transport: asyncio.Transport = None
self.writer: 'HashBlobWriter' = None
self.blob: 'BlobFile' = None
self.download_running = asyncio.Event(loop=self.loop)
self._blob_bytes_received = 0
self._response_fut: asyncio.Future = None
self._request_lock = asyncio.Lock(loop=self.loop)
def handle_data_received(self, data: bytes):
if self.transport.is_closing():
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()):
self._response_fut.cancel()
return
response = BlobResponse.deserialize(data)
if response.responses and self.blob:
blob_response = response.get_blob_response()
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
self.blob.set_length(blob_response.length)
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
return
if response.responses:
self._response_fut.set_result(response)
if response.blob_data and self.writer and not self.writer.closed():
self._blob_bytes_received += len(response.blob_data)
try:
self.writer.write(response.blob_data)
except IOError as err:
log.error("error downloading blob: %s", err)
def data_received(self, data):
try:
return self.handle_data_received(data)
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
async def _download_blob(self) -> typing.Tuple[bool, bool]:
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
try:
self.transport.write(request.serialize())
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
availability_response = response.get_availability_response()
price_response = response.get_price_response()
blob_response = response.get_blob_response()
if (not blob_response or blob_response.error) and\
(not availability_response or not availability_response.available_blobs):
log.warning("blob not in availability response")
return False, True
elif availability_response.available_blobs and \
availability_response.available_blobs != [self.blob.blob_hash]:
log.warning("blob availability response doesn't match our request")
return False, False
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
log.warning("data rate rejected")
return False, False
if not blob_response or blob_response.error:
log.warning("blob cant be downloaded from this peer")
return False, True
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
log.warning("incoming blob hash mismatch")
return False, False
if self.blob.length is not None and self.blob.length != blob_response.length:
log.warning("incoming blob unexpected length")
return False, False
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
f" timeout in {self.peer_timeout}"
log.info(msg)
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
await self.blob.finished_writing.wait()
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}"
log.info(msg)
return True, True
except asyncio.CancelledError:
return False, True
except asyncio.TimeoutError:
return False, False
finally:
await self.close()
async def close(self):
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
if self.writer and not self.writer.closed():
self.writer.close_handle()
if self.blob:
await self.blob.close()
self.download_running.clear()
self._response_fut = None
self.writer = None
self.blob = None
if self.transport:
self.transport.close()
self.transport = None
async def download_blob(self, blob: 'BlobFile') -> typing.Tuple[bool, bool]:
if blob.get_is_verified():
return False, True
async with self._request_lock:
try:
if self.download_running.is_set():
log.info("wait for download already running")
await self.download_running.wait()
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
self.download_running.set()
self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob()
except OSError:
log.error("race happened")
# i'm not sure how to fix this race condition - jack
return False, True
except asyncio.TimeoutError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return False, False
except asyncio.CancelledError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return False, True
def connection_made(self, transport: asyncio.Transport):
self.transport = transport
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
# log.info("connection made to %s: %s", self.peer_address, transport)
def connection_lost(self, reason):
# log.info("connection lost to %s (reason: %s)", self.peer_address, reason)
self.transport = None
self.loop.create_task(self.close())
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol: 'BlobExchangeClientProtocol',
address: str, tcp_port: int, peer_connect_timeout: float) -> typing.Tuple[bool, bool]:
"""
Returns [<downloaded blob>, <keep connection>]
"""
if blob.get_is_verified():
return False, True
if blob.get_is_verified():
log.info("already verified")
return False, True
try:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
return await protocol.download_blob(blob)
except (asyncio.TimeoutError, asyncio.CancelledError, ConnectionRefusedError, ConnectionAbortedError, OSError):
return False, False
-132
View File
@@ -1,132 +0,0 @@
import asyncio
import typing
import logging
from lbrynet import conf
from lbrynet.utils import drain_tasks
from lbrynet.blob_exchange.client import BlobExchangeClientProtocol, request_blob
if typing.TYPE_CHECKING:
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob.blob_file import BlobFile
log = logging.getLogger(__name__)
def drain_into(a: list, b: list):
while a:
b.append(a.pop())
class BlobDownloader: # TODO: refactor to be the base class used by StreamDownloader
"""A single blob downloader"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', config: conf.Config):
self.loop = loop
self.blob_manager = blob_manager
self.new_peer_event = asyncio.Event(loop=self.loop)
self.active_connections: typing.Dict['KademliaPeer', BlobExchangeClientProtocol] = {}
self.running_download_requests: typing.List[asyncio.Task] = []
self.requested_from: typing.Dict[str, typing.Dict['KademliaPeer', asyncio.Task]] = {}
self.lock = asyncio.Lock(loop=self.loop)
self.blob: 'BlobFile' = None
self.blob_queue = asyncio.Queue(loop=self.loop)
self.blob_download_timeout = config.get('blob_download_timeout')
self.peer_connect_timeout = config.get('peer_connect_timeout')
self.max_connections = config.get('max_connections_per_stream')
async def _request_blob(self, peer: 'KademliaPeer'):
if self.blob.get_is_verified():
log.info("already verified")
return
if peer not in self.active_connections:
log.warning("not active, adding: %s", str(peer))
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
protocol = self.active_connections[peer]
success, keep_connection = await request_blob(self.loop, self.blob, protocol, peer.address, peer.tcp_port,
self.peer_connect_timeout)
await protocol.close()
if not keep_connection:
log.info("drop peer %s:%i", peer.address, peer.tcp_port)
if peer in self.active_connections:
async with self.lock:
del self.active_connections[peer]
return
log.info("keep peer %s:%i", peer.address, peer.tcp_port)
def _update_requests(self):
self.new_peer_event.clear()
if self.blob.blob_hash not in self.requested_from:
self.requested_from[self.blob.blob_hash] = {}
to_add = []
for peer in self.active_connections.keys():
if peer not in self.requested_from[self.blob.blob_hash] and peer not in to_add:
to_add.append(peer)
if to_add or self.running_download_requests:
log.info("adding download probes for %i peers to %i already active",
min(len(to_add), 8 - len(self.running_download_requests)),
len(self.running_download_requests))
else:
log.info("downloader idle...")
for peer in to_add:
if len(self.running_download_requests) >= 8:
break
task = self.loop.create_task(self._request_blob(peer))
self.requested_from[self.blob.blob_hash][peer] = task
self.running_download_requests.append(task)
def _add_peer_protocols(self, peers: typing.List['KademliaPeer']):
added = 0
for peer in peers:
if peer not in self.active_connections:
self.active_connections[peer] = BlobExchangeClientProtocol(self.loop, self.blob_download_timeout)
added += 1
if added:
if not self.new_peer_event.is_set():
log.info("added %i new peers", len(peers))
self.new_peer_event.set()
async def _accumulate_connections(self, node: 'Node'):
try:
async with node.stream_peer_search_junction(self.blob_queue) as search_junction:
async for peers in search_junction:
if not isinstance(peers, list): # TODO: what's up with this?
log.error("not a list: %s %s", peers, str(type(peers)))
else:
self._add_peer_protocols(peers)
return
except asyncio.CancelledError:
pass
async def get_blob(self, blob_hash: str, node: 'Node') -> 'BlobFile':
self.blob = self.blob_manager.get_blob(blob_hash)
if self.blob.get_is_verified():
return self.blob
accumulator = self.loop.create_task(self._accumulate_connections(node))
self.blob_queue.put_nowait(blob_hash)
try:
while not self.blob.get_is_verified():
if len(self.running_download_requests) < self.max_connections:
self._update_requests()
# drain the tasks into a temporary list
download_tasks = []
drain_into(self.running_download_requests, download_tasks)
got_new_peer = self.loop.create_task(self.new_peer_event.wait())
# wait for a new peer to be added or for a download attempt to finish
await asyncio.wait([got_new_peer] + download_tasks, return_when='FIRST_COMPLETED',
loop=self.loop)
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
if self.blob.get_is_verified():
if got_new_peer and not got_new_peer.done():
got_new_peer.cancel()
drain_tasks(download_tasks)
return self.blob
except asyncio.CancelledError:
drain_tasks(self.running_download_requests)
raise
finally:
if accumulator and not accumulator.done():
accumulator.cancel()
-289
View File
@@ -1,289 +0,0 @@
import typing
import json
import logging
log = logging.getLogger(__name__)
class BlobMessage:
key = ''
def to_dict(self) -> typing.Dict:
raise NotImplementedError()
class BlobPriceRequest(BlobMessage):
key = 'blob_data_payment_rate'
def __init__(self, blob_data_payment_rate: float, **kwargs) -> None:
self.blob_data_payment_rate = blob_data_payment_rate
def to_dict(self) -> typing.Dict:
return {
self.key: self.blob_data_payment_rate
}
class BlobPriceResponse(BlobMessage):
key = 'blob_data_payment_rate'
rate_accepted = 'RATE_ACCEPTED'
rate_too_low = 'RATE_TOO_LOW'
rate_unset = 'RATE_UNSET'
def __init__(self, blob_data_payment_rate: str, **kwargs) -> None:
if blob_data_payment_rate not in (self.rate_accepted, self.rate_too_low, self.rate_unset):
raise ValueError(blob_data_payment_rate)
self.blob_data_payment_rate = blob_data_payment_rate
def to_dict(self) -> typing.Dict:
return {
self.key: self.blob_data_payment_rate
}
class BlobAvailabilityRequest(BlobMessage):
key = 'requested_blobs'
def __init__(self, requested_blobs: typing.List[str], lbrycrd_address: typing.Optional[bool] = True,
**kwargs) -> None:
assert len(requested_blobs)
self.requested_blobs = requested_blobs
self.lbrycrd_address = lbrycrd_address
def to_dict(self) -> typing.Dict:
return {
self.key: self.requested_blobs,
'lbrycrd_address': self.lbrycrd_address
}
class BlobAvailabilityResponse(BlobMessage):
key = 'available_blobs'
def __init__(self, available_blobs: typing.List[str], lbrycrd_address: typing.Optional[str] = True,
**kwargs) -> None:
self.available_blobs = available_blobs
self.lbrycrd_address = lbrycrd_address
def to_dict(self) -> typing.Dict:
d = {
self.key: self.available_blobs
}
if self.lbrycrd_address:
d['lbrycrd_address'] = self.lbrycrd_address
return d
class BlobDownloadRequest(BlobMessage):
key = 'requested_blob'
def __init__(self, requested_blob: str, **kwargs) -> None:
self.requested_blob = requested_blob
def to_dict(self) -> typing.Dict:
return {
self.key: self.requested_blob
}
class BlobDownloadResponse(BlobMessage):
key = 'incoming_blob'
def __init__(self, **response: typing.Dict) -> None:
incoming_blob = response[self.key]
self.error = None
self.incoming_blob = None
if 'error' in incoming_blob:
self.error = incoming_blob['error']
else:
self.incoming_blob = {'blob_hash': incoming_blob['blob_hash'], 'length': incoming_blob['length']}
self.length = None if not self.incoming_blob else self.incoming_blob['length']
self.blob_hash = None if not self.incoming_blob else self.incoming_blob['blob_hash']
def to_dict(self) -> typing.Dict:
return {
self.key if not self.error else 'error': self.incoming_blob or self.error,
}
class BlobPaymentAddressRequest(BlobMessage):
key = 'lbrycrd_address'
def __init__(self, lbrycrd_address: str, **kwargs) -> None:
self.lbrycrd_address = lbrycrd_address
def to_dict(self) -> typing.Dict:
return {
self.key: self.lbrycrd_address
}
class BlobPaymentAddressResponse(BlobPaymentAddressRequest):
pass
class BlobErrorResponse(BlobMessage):
key = 'error'
def __init__(self, error: str, **kwargs) -> None:
self.error = error
def to_dict(self) -> typing.Dict:
return {
self.key: self.error
}
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
BlobPaymentAddressRequest]
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
BlobErrorResponse, BlobPaymentAddressResponse]
def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[typing.Dict], bytes]:
# scenarios:
# <json>
# <blob bytes>
# <json><blob bytes>
extra_data = b''
response = None
curr_pos = 0
while True:
next_close_paren = response_msg.find(b'}', curr_pos)
if next_close_paren == -1:
break
curr_pos = next_close_paren + 1
try:
response = json.loads(response_msg[:curr_pos])
if not isinstance(response, dict):
raise ValueError()
for key in response.keys():
if key not in [
BlobPaymentAddressResponse.key,
BlobAvailabilityResponse.key,
BlobPriceResponse.key,
BlobDownloadResponse.key]:
raise ValueError()
extra_data = response_msg[curr_pos:]
break
except ValueError:
response = None
if response is None:
extra_data = response_msg
return response, extra_data
class BlobRequest:
def __init__(self, requests: typing.List[blob_request_types]) -> None:
self.requests = requests
def to_dict(self):
d = {}
for request in self.requests:
d.update(request.to_dict())
return d
def _get_request(self, request_type: blob_request_types):
request = tuple(filter(lambda r: type(r) == request_type, self.requests))
if request:
return request[0]
def get_availability_request(self) -> typing.Optional[BlobAvailabilityRequest]:
response = self._get_request(BlobAvailabilityRequest)
if response:
return response
def get_price_request(self) -> typing.Optional[BlobPriceRequest]:
response = self._get_request(BlobPriceRequest)
if response:
return response
def get_blob_request(self) -> typing.Optional[BlobDownloadRequest]:
response = self._get_request(BlobDownloadRequest)
if response:
return response
def get_address_request(self) -> typing.Optional[BlobPaymentAddressRequest]:
response = self._get_request(BlobPaymentAddressRequest)
if response:
return response
def serialize(self) -> bytes:
return json.dumps(self.to_dict()).encode()
@classmethod
def deserialize(cls, data: bytes) -> 'BlobRequest':
request = json.loads(data)
return cls([
request_type(**request)
for request_type in (BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
BlobPaymentAddressRequest)
if request_type.key in request
])
@classmethod
def make_request_for_blob_hash(cls, blob_hash: str) -> 'BlobRequest':
return cls(
[BlobAvailabilityRequest([blob_hash]), BlobPriceRequest(0.0), BlobDownloadRequest(blob_hash)]
)
class BlobResponse:
def __init__(self, responses: typing.List[blob_response_types], blob_data: typing.Optional[bytes] = None) -> None:
self.responses = responses
self.blob_data = blob_data
def to_dict(self):
d = {}
for response in self.responses:
d.update(response.to_dict())
return d
def _get_response(self, response_type: blob_response_types):
response = tuple(filter(lambda r: type(r) == response_type, self.responses))
if response:
return response[0]
def get_error_response(self) -> typing.Optional[BlobErrorResponse]:
error = self._get_response(BlobErrorResponse)
if error:
log.error(error)
return error
def get_availability_response(self) -> typing.Optional[BlobAvailabilityResponse]:
response = self._get_response(BlobAvailabilityResponse)
if response:
return response
def get_price_response(self) -> typing.Optional[BlobPriceResponse]:
response = self._get_response(BlobPriceResponse)
if response:
return response
def get_blob_response(self) -> typing.Optional[BlobDownloadResponse]:
response = self._get_response(BlobDownloadResponse)
if response:
return response
def get_address_response(self) -> typing.Optional[BlobPaymentAddressResponse]:
response = self._get_response(BlobPaymentAddressResponse)
if response:
return response
def serialize(self) -> bytes:
return json.dumps(self.to_dict()).encode()
@classmethod
def deserialize(cls, data: bytes) -> 'BlobResponse':
response, extra = _parse_blob_response(data)
requests = []
if response:
requests.extend([
response_type(**response)
for response_type in (BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
BlobErrorResponse, BlobPaymentAddressResponse)
if response_type.key in response
])
return cls(requests, extra)
-119
View File
@@ -1,119 +0,0 @@
import asyncio
import binascii
import logging
import typing
from json.decoder import JSONDecodeError
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPriceResponse, BlobDownloadResponse, \
BlobPaymentAddressResponse
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_manager import BlobFileManager
log = logging.getLogger(__name__)
class BlobServerProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
self.started_listening = asyncio.Event(loop=self.loop)
self.buf = b''
self.transport = None
self.lbrycrd_address = lbrycrd_address
def connection_made(self, transport):
self.transport = transport
def send_response(self, responses: typing.List[blob_response_types]):
to_send = []
while responses:
to_send.append(responses.pop())
self.transport.write(BlobResponse(to_send).serialize())
async def handle_request(self, request: BlobRequest):
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
responses = []
address_request = request.get_address_request()
if address_request:
responses.append(BlobPaymentAddressResponse(lbrycrd_address=self.lbrycrd_address))
availability_request = request.get_availability_request()
if availability_request:
responses.append(BlobAvailabilityResponse(available_blobs=list(set((
filter(lambda blob_hash: blob_hash in self.blob_manager.completed_blob_hashes,
availability_request.requested_blobs)
)))))
price_request = request.get_price_request()
if price_request:
responses.append(BlobPriceResponse(blob_data_payment_rate='RATE_ACCEPTED'))
download_request = request.get_blob_request()
if download_request:
blob = self.blob_manager.get_blob(download_request.requested_blob)
if blob.get_is_verified():
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
self.send_response(responses)
log.info("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
sent = await blob.sendfile(self)
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
if responses:
self.send_response(responses)
# self.transport.close()
def data_received(self, data):
request = None
if data:
message, separator, remainder = data.rpartition(b'}')
if not separator:
self.buf += data
return
try:
request = BlobRequest.deserialize(data)
self.buf = remainder
except JSONDecodeError:
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
log.error("failed to decode blob request from %s:%i (%i bytes): %s", peer_address, peer_port,
len(data), '' if not data else binascii.hexlify(data).decode())
if not request:
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
log.warning("failed to decode blob request from %s:%i", peer_address, peer_port)
self.transport.close()
return
self.loop.create_task(self.handle_request(request))
class BlobServer:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobFileManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
self.started_listening = asyncio.Event(loop=self.loop)
self.lbrycrd_address = lbrycrd_address
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
raise Exception("already running")
async def _start_server():
server = await self.loop.create_server(
lambda: BlobServerProtocol(self.loop, self.blob_manager, self.lbrycrd_address),
interface, port
)
self.started_listening.set()
log.info("Blob server listening on TCP %s:%i", interface, port)
async with server:
await server.serve_forever()
self.server_task = self.loop.create_task(_start_server())
def stop_server(self):
if self.server_task:
self.server_task.cancel()
self.server_task = None
log.info("Stopped blob server")
+1 -1
View File
@@ -1,2 +1,2 @@
# don't touch this. CI server changes this during build/deployment
# dont touch this. CI server changes this during build/deployment
BUILD = "dev"
+585 -553
View File
File diff suppressed because it is too large. Load diff
+111
View File
@@ -0,0 +1,111 @@
import logging
import random
import time
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from decimal import Decimal
log = logging.getLogger(__name__)
class BlobAvailabilityTracker(object):
"""
Class to track peer counts for known blobs, and to discover new popular blobs
Attributes:
availability (dict): dictionary of peers for known blobs
"""
def __init__(self, blob_manager, peer_finder, dht_node):
self.availability = {}
self._last_mean_availability = Decimal(0.0)
self._blob_manager = blob_manager
self._peer_finder = peer_finder
self._dht_node = dht_node
self._check_popular = LoopingCall(self._update_most_popular)
self._check_mine = LoopingCall(self._update_mine)
def start(self):
log.info("Starting blob availability tracker.")
self._check_popular.start(600)
self._check_mine.start(600)
def stop(self):
log.info("Stopping blob availability tracker.")
if self._check_popular.running:
self._check_popular.stop()
if self._check_mine.running:
self._check_mine.stop()
def get_blob_availability(self, blob, timeout=None):
def _get_peer_count(peers):
have_blob = sum(1 for peer in peers if peer.is_available())
return {blob: have_blob}
d = self._peer_finder.find_peers_for_blob(blob, timeout)
d.addCallback(_get_peer_count)
return d
def get_availability_for_blobs(self, blobs, timeout=None):
dl = [self.get_blob_availability(blob, timeout) for blob in blobs if blob]
d = defer.DeferredList(dl)
d.addCallback(lambda results: [val for success, val in results if success])
return d
@property
def last_mean_availability(self):
return max(Decimal(0.01), self._last_mean_availability)
def _update_peers_for_blob(self, blob):
def _save_peer_info(blob_hash, peers):
v = {blob_hash: peers}
self.availability.update(v)
return v
d = self._peer_finder.find_peers_for_blob(blob)
d.addCallback(lambda r: [[c.host, c.port, c.is_available()] for c in r])
d.addCallback(lambda peers: _save_peer_info(blob, peers))
return d
def _get_most_popular(self):
dl = []
for (hash, _) in self._dht_node.get_most_popular_hashes(10):
encoded = hash.encode('hex')
dl.append(self._update_peers_for_blob(encoded))
return defer.DeferredList(dl)
def _update_most_popular(self):
d = self._get_most_popular()
d.addCallback(lambda _: self._set_mean_peers())
def _update_mine(self):
def _get_peers(blobs):
dl = []
for hash in blobs:
dl.append(self._update_peers_for_blob(hash))
return defer.DeferredList(dl)
def sample(blobs):
return random.sample(blobs, min(len(blobs), 10))
start = time.time()
log.debug('==> Updating the peers for my blobs')
d = self._blob_manager.get_all_verified_blobs()
# as far as I can tell, this only is used to set _last_mean_availability
# which... seems like a very expensive operation for such little payoff.
# so taking a sample should get about the same effect as querying the entire
# list of blobs
d.addCallback(sample)
d.addCallback(_get_peers)
d.addCallback(lambda _: self._set_mean_peers())
d.addCallback(lambda _: log.debug('<== Done updating peers for my blobs. Took %s seconds',
time.time() - start))
# although unused, need to return or else the looping call
# could overrun on a previous call
return d
def _set_mean_peers(self):
num_peers = [len(self.availability[blob]) for blob in self.availability]
mean = Decimal(sum(num_peers)) / Decimal(max(1, len(num_peers)))
self._last_mean_availability = mean
+19
View File
@@ -0,0 +1,19 @@
class BlobInfo(object):
"""
This structure is used to represent the metadata of a blob.
@ivar blob_hash: The sha384 hashsum of the blob's data.
@type blob_hash: string, hex-encoded
@ivar blob_num: For streams, the position of the blob in the stream.
@type blob_num: integer
@ivar length: The length of the blob in bytes.
@type length: integer
"""
def __init__(self, blob_hash, blob_num, length):
self.blob_hash = blob_hash
self.blob_num = blob_num
self.length = length
+162
View File
@@ -0,0 +1,162 @@
import logging
import os
from sqlite3 import IntegrityError
from twisted.internet import threads, defer, reactor, task
from lbrynet import conf
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.creator import BlobFileCreator
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
log = logging.getLogger(__name__)
class DiskBlobManager(DHTHashSupplier):
def __init__(self, hash_announcer, blob_dir, storage):
"""
This class stores blobs on the hard disk,
blob_dir - directory where blobs are stored
db_dir - directory where sqlite database of blob information is stored
"""
DHTHashSupplier.__init__(self, hash_announcer)
self.storage = storage
self.announce_head_blobs_only = conf.settings['announce_head_blobs_only']
self.blob_dir = blob_dir
self.blob_creator_type = BlobFileCreator
# TODO: consider using an LRU for blobs as there could potentially
# be thousands of blobs loaded up, many stale
self.blobs = {}
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
self.check_should_announce_lc = None
if conf.settings['run_reflector_server']:
self.check_should_announce_lc = task.LoopingCall(self.storage.verify_will_announce_all_head_and_sd_blobs)
def setup(self):
if self.check_should_announce_lc and not self.check_should_announce_lc.running:
self.check_should_announce_lc.start(600)
return defer.succeed(True)
def stop(self):
if self.check_should_announce_lc and self.check_should_announce_lc.running:
self.check_should_announce_lc.stop()
return defer.succeed(True)
def get_blob(self, blob_hash, length=None):
"""Return a blob identified by blob_hash, which may be a new blob or a
blob that is already on the hard disk
"""
if length is not None and not isinstance(length, int):
raise Exception("invalid length type: %s (%s)" % (length, str(type(length))))
if blob_hash in self.blobs:
return defer.succeed(self.blobs[blob_hash])
return self._make_new_blob(blob_hash, length)
def get_blob_creator(self):
return self.blob_creator_type(self.blob_dir)
def _make_new_blob(self, blob_hash, length=None):
log.debug('Making a new blob for %s', blob_hash)
blob = BlobFile(self.blob_dir, blob_hash, length)
self.blobs[blob_hash] = blob
return defer.succeed(blob)
def immediate_announce(self, blob_hashes):
if self.hash_announcer:
return self.hash_announcer.immediate_announce(blob_hashes)
raise Exception("Hash announcer not set")
@defer.inlineCallbacks
def blob_completed(self, blob, next_announce_time=None, should_announce=True):
if next_announce_time is None:
next_announce_time = self.get_next_announce_time()
yield self.storage.add_completed_blob(
blob.blob_hash, blob.length, next_announce_time, should_announce
)
# we announce all blobs immediately, if announce_head_blob_only is False
# otherwise, announce only if marked as should_announce
if not self.announce_head_blobs_only or should_announce:
reactor.callLater(0, self.immediate_announce, [blob.blob_hash])
def completed_blobs(self, blobhashes_to_check):
return self._completed_blobs(blobhashes_to_check)
def hashes_to_announce(self):
return self.storage.get_blobs_to_announce(self.hash_announcer)
def count_should_announce_blobs(self):
return self.storage.count_should_announce_blobs()
def set_should_announce(self, blob_hash, should_announce):
if blob_hash in self.blobs:
blob = self.blobs[blob_hash]
if blob.get_is_verified():
return self.storage.set_should_announce(
blob_hash, self.get_next_announce_time(), should_announce
)
return defer.succeed(False)
def get_should_announce(self, blob_hash):
return self.storage.should_announce(blob_hash)
def creator_finished(self, blob_creator, should_announce):
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
if blob_creator.blob_hash is None:
raise Exception("Blob hash is None")
if blob_creator.blob_hash in self.blobs:
raise Exception("Creator finished for blob that is already marked as completed")
if blob_creator.length is None:
raise Exception("Blob has a length of 0")
new_blob = BlobFile(self.blob_dir, blob_creator.blob_hash, blob_creator.length)
self.blobs[blob_creator.blob_hash] = new_blob
next_announce_time = self.get_next_announce_time()
return self.blob_completed(new_blob, next_announce_time, should_announce)
def immediate_announce_all_blobs(self):
d = self._get_all_verified_blob_hashes()
d.addCallback(self.immediate_announce)
return d
def get_all_verified_blobs(self):
d = self._get_all_verified_blob_hashes()
d.addCallback(self.completed_blobs)
return d
@defer.inlineCallbacks
def delete_blobs(self, blob_hashes):
bh_to_delete_from_db = []
for blob_hash in blob_hashes:
try:
blob = yield self.get_blob(blob_hash)
yield blob.delete()
bh_to_delete_from_db.append(blob_hash)
del self.blobs[blob_hash]
except Exception as e:
log.warning("Failed to delete blob file. Reason: %s", e)
try:
yield self.storage.delete_blobs_from_db(bh_to_delete_from_db)
except IntegrityError as err:
if err.message != "FOREIGN KEY constraint failed":
raise err
@defer.inlineCallbacks
def _completed_blobs(self, blobhashes_to_check):
"""Returns of the blobhashes_to_check, which are valid"""
blobs = yield defer.DeferredList([self.get_blob(b) for b in blobhashes_to_check])
blob_hashes = [b.blob_hash for success, b in blobs if success and b.verified]
defer.returnValue(blob_hashes)
def _get_all_verified_blob_hashes(self):
d = self.storage.get_all_blob_hashes()
def get_verified_blobs(blobs):
verified_blobs = []
for blob_hash in blobs:
file_path = os.path.join(self.blob_dir, blob_hash)
if os.path.isfile(file_path):
verified_blobs.append(blob_hash)
return verified_blobs
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
return d
+21
View File
@@ -0,0 +1,21 @@
class DownloadOptionChoice(object):
"""A possible choice that can be picked for some option.
An option can have one or more choices that can be picked from.
"""
def __init__(self, value, short_description, long_description, bool_options_description=None):
self.value = value
self.short_description = short_description
self.long_description = long_description
self.bool_options_description = bool_options_description
class DownloadOption(object):
"""An option for a user to select a value from several different choices."""
def __init__(self, option_types, long_description, short_description, default_value,
default_value_description):
self.option_types = option_types
self.long_description = long_description
self.short_description = short_description
self.default_value = default_value
self.default_value_description = default_value_description
+17 -47
View File
@@ -1,7 +1,3 @@
class RPCError(Exception):
code = 0
class PriceDisagreementError(Exception):
pass
@@ -10,25 +6,25 @@ class DuplicateStreamHashError(Exception):
pass
class DownloadCancelledError(Exception):
class DownloadCanceledError(Exception):
pass
class DownloadSDTimeout(Exception):
def __init__(self, download):
super().__init__(f'Failed to download sd blob {download} within timeout')
Exception.__init__(self, 'Failed to download sd blob {} within timeout'.format(download))
self.download = download
class DownloadTimeoutError(Exception):
def __init__(self, download):
super().__init__(f'Failed to download {download} within timeout')
Exception.__init__(self, 'Failed to download {} within timeout'.format(download))
self.download = download
class DownloadDataTimeout(Exception):
def __init__(self, download):
super().__init__('Failed to download data blobs for sd hash '
Exception.__init__(self, 'Failed to download data blobs for sd hash '
'{} within timeout'.format(download))
self.download = download
@@ -45,21 +41,7 @@ class NullFundsError(Exception):
pass
class InsufficientFundsError(RPCError):
code = -310
class CurrencyConversionError(Exception):
pass
class FileOpenError(ValueError):
# this extends ValueError because it is replacing a ValueError in EncryptedFileDownloader
# and I don't know where it might get caught upstream
pass
class ResolveError(Exception):
class InsufficientFundsError(Exception):
pass
@@ -73,41 +55,39 @@ class KeyFeeAboveMaxAllowed(Exception):
class InvalidExchangeRateResponse(Exception):
def __init__(self, source, reason):
super().__init__(f'Failed to get exchange rate from {source}:{reason}')
Exception.__init__(self, 'Failed to get exchange rate from {}:{}'.format(source, reason))
self.source = source
self.reason = reason
class UnknownNameError(Exception):
def __init__(self, name):
super().__init__(f'Name {name} is unknown')
Exception.__init__(self, 'Name {} is unknown'.format(name))
self.name = name
class UnknownClaimID(Exception):
def __init__(self, claim_id):
super().__init__(f'Claim {claim_id} is unknown')
Exception.__init__(self, 'Claim {} is unknown'.format(claim_id))
self.claim_id = claim_id
class UnknownURI(Exception):
def __init__(self, uri):
super().__init__(f'URI {uri} cannot be resolved')
Exception.__init__(self, 'URI {} cannot be resolved'.format(uri))
self.name = uri
class UnknownOutpoint(Exception):
def __init__(self, outpoint):
super().__init__(f'Outpoint {outpoint} cannot be resolved')
Exception.__init__(self, 'Outpoint {} cannot be resolved'.format(outpoint))
self.outpoint = outpoint
class InvalidName(Exception):
def __init__(self, name, invalid_characters):
self.name = name
self.invalid_characters = invalid_characters
super().__init__(
'URI contains invalid characters: {}'.format(','.join(invalid_characters)))
Exception.__init__(
self, 'URI contains invalid characters: {}'.format(','.join(invalid_characters)))
class UnknownStreamTypeError(Exception):
@@ -124,8 +104,8 @@ class InvalidStreamDescriptorError(Exception):
class InvalidStreamInfoError(Exception):
def __init__(self, name, stream_info):
msg = f'{name} has claim with invalid stream info: {stream_info}'
super().__init__(msg)
msg = '{} has claim with invalid stream info: {}'.format(name, stream_info)
Exception.__init__(self, msg)
self.name = name
self.stream_info = stream_info
@@ -175,23 +155,13 @@ class InvalidAuthenticationToken(Exception):
class NegotiationError(Exception):
pass
class InvalidCurrencyError(Exception):
def __init__(self, currency):
self.currency = currency
super().__init__(
f'Invalid currency: {currency} is not a supported currency.')
Exception.__init__(
self, 'Invalid currency: {} is not a supported currency.'.format(currency))
class NoSuchDirectoryError(Exception):
def __init__(self, directory):
self.directory = directory
super().__init__(f'No such directory {directory}')
class ComponentStartConditionNotMet(Exception):
pass
class ComponentsNotStarted(Exception):
pass
Exception.__init__(self, 'No such directory {}'.format(directory))
+18
View File
@@ -0,0 +1,18 @@
class DummyHashAnnouncer(object):
def __init__(self, *args):
pass
def run_manage_loop(self):
pass
def stop(self):
pass
def add_supplier(self, *args):
pass
def hash_queue_size(self):
return 0
def immediate_announce(self, *args):
pass
+62
View File
@@ -0,0 +1,62 @@
from decimal import Decimal
class Offer(object):
"""A rate offer to download blobs from a host."""
RATE_ACCEPTED = "RATE_ACCEPTED"
RATE_TOO_LOW = "RATE_TOO_LOW"
RATE_UNSET = "RATE_UNSET"
def __init__(self, offer):
self._state = None
self.rate = None
if isinstance(offer, Decimal):
self.rate = round(offer, 5)
elif isinstance(offer, float):
self.rate = round(Decimal(offer), 5)
if self.rate is None or self.rate < Decimal(0.0):
self.unset()
@property
def is_accepted(self):
return self._state is Offer.RATE_ACCEPTED
@property
def is_too_low(self):
return self._state is Offer.RATE_TOO_LOW
@property
def is_unset(self):
return self._state is Offer.RATE_UNSET
@property
def message(self):
if self.is_accepted:
return Offer.RATE_ACCEPTED
elif self.is_too_low:
return Offer.RATE_TOO_LOW
elif self.is_unset:
return Offer.RATE_UNSET
return None
def accept(self):
if self.is_unset or self._state is None:
self._state = Offer.RATE_ACCEPTED
def reject(self):
if self.is_unset or self._state is None:
self._state = Offer.RATE_TOO_LOW
def unset(self):
self._state = Offer.RATE_UNSET
def handle(self, reply_message):
if reply_message == Offer.RATE_TOO_LOW:
self.reject()
elif reply_message == Offer.RATE_ACCEPTED:
self.accept()
elif reply_message == Offer.RATE_UNSET:
self.unset()
else:
raise Exception("Unknown offer reply %s" % str(reply_message))
+330
View File
@@ -0,0 +1,330 @@
from collections import defaultdict
import logging
import os
import unqlite
import time
from Crypto.Hash import SHA512
from Crypto.PublicKey import RSA
from lbrynet.core.client.ClientRequest import ClientRequest
from lbrynet.core.Error import RequestCanceledError
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, IWallet
from lbrynet.pointtraderclient import pointtraderclient
from twisted.internet import defer, threads
from zope.interface import implements
from twisted.python.failure import Failure
from lbrynet.core.Wallet import ReservedPoints
log = logging.getLogger(__name__)
class PTCWallet(object):
"""This class sends payments to peers and also ensures that expected payments are received.
This class is only intended to be used for testing."""
implements(IWallet)
def __init__(self, db_dir):
self.db_dir = db_dir
self.db = None
self.private_key = None
self.encoded_public_key = None
self.peer_pub_keys = {}
self.queued_payments = defaultdict(int)
self.expected_payments = defaultdict(list)
self.received_payments = defaultdict(list)
self.next_manage_call = None
self.payment_check_window = 3 * 60 # 3 minutes
self.new_payments_expected_time = time.time() - self.payment_check_window
self.known_transactions = []
self.total_reserved_points = 0.0
self.wallet_balance = 0.0
def manage(self):
"""Send payments, ensure expected payments are received"""
from twisted.internet import reactor
if time.time() < self.new_payments_expected_time + self.payment_check_window:
d1 = self._get_new_payments()
else:
d1 = defer.succeed(None)
d1.addCallback(lambda _: self._check_good_standing())
d2 = self._send_queued_points()
self.next_manage_call = reactor.callLater(60, self.manage)
dl = defer.DeferredList([d1, d2])
dl.addCallback(lambda _: self.get_balance())
def set_balance(balance):
self.wallet_balance = balance
dl.addCallback(set_balance)
return dl
def stop(self):
if self.next_manage_call is not None:
self.next_manage_call.cancel()
self.next_manage_call = None
d = self.manage()
self.next_manage_call.cancel()
self.next_manage_call = None
self.db = None
return d
def start(self):
def save_key(success, private_key):
if success is True:
self._save_private_key(private_key.exportKey())
return True
return False
def register_private_key(private_key):
self.private_key = private_key
self.encoded_public_key = self.private_key.publickey().exportKey()
d_r = pointtraderclient.register_new_account(private_key)
d_r.addCallback(save_key, private_key)
return d_r
def ensure_private_key_exists(encoded_private_key):
if encoded_private_key is not None:
self.private_key = RSA.importKey(encoded_private_key)
self.encoded_public_key = self.private_key.publickey().exportKey()
return True
else:
create_d = threads.deferToThread(RSA.generate, 4096)
create_d.addCallback(register_private_key)
return create_d
def start_manage():
self.manage()
return True
d = self._open_db()
d.addCallback(lambda _: self._get_wallet_private_key())
d.addCallback(ensure_private_key_exists)
d.addCallback(lambda _: start_manage())
return d
def get_info_exchanger(self):
return PointTraderKeyExchanger(self)
def get_wallet_info_query_handler_factory(self):
return PointTraderKeyQueryHandlerFactory(self)
def reserve_points(self, peer, amount):
"""Ensure a certain amount of points are available to be sent as
payment, before the service is rendered
@param peer: The peer to which the payment will ultimately be sent
@param amount: The amount of points to reserve
@return: A ReservedPoints object which is given to send_points
once the service has been rendered
"""
if self.wallet_balance >= self.total_reserved_points + amount:
self.total_reserved_points += amount
return ReservedPoints(peer, amount)
return None
def cancel_point_reservation(self, reserved_points):
"""
Return all of the points that were reserved previously for some ReservedPoints object
@param reserved_points: ReservedPoints previously returned by reserve_points
@return: None
"""
self.total_reserved_points -= reserved_points.amount
def send_points(self, reserved_points, amount):
"""
Schedule a payment to be sent to a peer
@param reserved_points: ReservedPoints object previously returned by reserve_points
@param amount: amount of points to actually send, must be less than or equal to the
amount reserved in reserved_points
@return: Deferred which fires when the payment has been scheduled
"""
self.queued_payments[reserved_points.identifier] += amount
# make any unused points available
self.total_reserved_points -= reserved_points.amount - amount
reserved_points.identifier.update_stats('points_sent', amount)
d = defer.succeed(True)
return d
def _send_queued_points(self):
ds = []
for peer, points in self.queued_payments.items():
if peer in self.peer_pub_keys:
d = pointtraderclient.send_points(
self.private_key, self.peer_pub_keys[peer], points)
self.wallet_balance -= points
self.total_reserved_points -= points
ds.append(d)
del self.queued_payments[peer]
else:
log.warning("Don't have a payment address for peer %s. Can't send %s points.",
str(peer), str(points))
return defer.DeferredList(ds)
def get_balance(self):
"""Return the balance of this wallet"""
d = pointtraderclient.get_balance(self.private_key)
return d
def add_expected_payment(self, peer, amount):
"""Increase the number of points expected to be paid by a peer"""
self.expected_payments[peer].append((amount, time.time()))
self.new_payments_expected_time = time.time()
peer.update_stats('expected_points', amount)
def set_public_key_for_peer(self, peer, pub_key):
self.peer_pub_keys[peer] = pub_key
def _get_new_payments(self):
def add_new_transactions(transactions):
for transaction in transactions:
if transaction[1] == self.encoded_public_key:
t_hash = SHA512.new()
t_hash.update(transaction[0])
t_hash.update(transaction[1])
t_hash.update(str(transaction[2]))
t_hash.update(transaction[3])
if t_hash.hexdigest() not in self.known_transactions:
self.known_transactions.append(t_hash.hexdigest())
self._add_received_payment(transaction[0], transaction[2])
d = pointtraderclient.get_recent_transactions(self.private_key)
d.addCallback(add_new_transactions)
return d
def _add_received_payment(self, encoded_other_public_key, amount):
self.received_payments[encoded_other_public_key].append((amount, time.time()))
def _check_good_standing(self):
for peer, expected_payments in self.expected_payments.iteritems():
expected_cutoff = time.time() - 90
min_expected_balance = sum([a[0] for a in expected_payments if a[1] < expected_cutoff])
received_balance = 0
if self.peer_pub_keys[peer] in self.received_payments:
received_balance = sum([
a[0] for a in self.received_payments[self.peer_pub_keys[peer]]])
if min_expected_balance > received_balance:
log.warning(
"Account in bad standing: %s (pub_key: %s), expected amount = %s, "
"received_amount = %s",
peer, self.peer_pub_keys[peer], min_expected_balance, received_balance)
def _open_db(self):
def open_db():
self.db = unqlite.UnQLite(os.path.join(self.db_dir, "ptcwallet.db"))
return threads.deferToThread(open_db)
def _save_private_key(self, private_key):
def save_key():
self.db['private_key'] = private_key
return threads.deferToThread(save_key)
def _get_wallet_private_key(self):
def get_key():
if 'private_key' in self.db:
return self.db['private_key']
return None
return threads.deferToThread(get_key)
class PointTraderKeyExchanger(object):
implements([IRequestCreator])
def __init__(self, wallet):
self.wallet = wallet
self._protocols = []
######### IRequestCreator #########
def send_next_request(self, peer, protocol):
if not protocol in self._protocols:
r = ClientRequest({'public_key': self.wallet.encoded_public_key},
'public_key')
d = protocol.add_request(r)
d.addCallback(self._handle_exchange_response, peer, r, protocol)
d.addErrback(self._request_failed, peer)
self._protocols.append(protocol)
return defer.succeed(True)
else:
return defer.succeed(False)
######### internal calls #########
def _handle_exchange_response(self, response_dict, peer, request, protocol):
assert request.response_identifier in response_dict, \
"Expected %s in dict but did not get it" % request.response_identifier
assert protocol in self._protocols, "Responding protocol is not in our list of protocols"
peer_pub_key = response_dict[request.response_identifier]
self.wallet.set_public_key_for_peer(peer, peer_pub_key)
return True
def _request_failed(self, err, peer):
if not err.check(RequestCanceledError):
log.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
err.getErrorMessage(), str(peer))
return err
class PointTraderKeyQueryHandlerFactory(object):
implements(IQueryHandlerFactory)
def __init__(self, wallet):
self.wallet = wallet
######### IQueryHandlerFactory #########
def build_query_handler(self):
q_h = PointTraderKeyQueryHandler(self.wallet)
return q_h
def get_primary_query_identifier(self):
return 'public_key'
def get_description(self):
return ("Point Trader Address - an address for receiving payments on the "
"point trader testing network")
class PointTraderKeyQueryHandler(object):
implements(IQueryHandler)
def __init__(self, wallet):
self.wallet = wallet
self.query_identifiers = ['public_key']
self.public_key = None
self.peer = None
######### IQueryHandler #########
def register_with_request_handler(self, request_handler, peer):
self.peer = peer
request_handler.register_query_handler(self, self.query_identifiers)
def handle_queries(self, queries):
if self.query_identifiers[0] in queries:
new_encoded_pub_key = queries[self.query_identifiers[0]]
try:
RSA.importKey(new_encoded_pub_key)
except (ValueError, TypeError, IndexError):
log.warning("Client sent an invalid public key.")
return defer.fail(Failure(ValueError("Client sent an invalid public key")))
self.public_key = new_encoded_pub_key
self.wallet.set_public_key_for_peer(self.peer, self.public_key)
log.debug("Received the client's public key: %s", str(self.public_key))
fields = {'public_key': self.wallet.encoded_public_key}
return defer.succeed(fields)
if self.public_key is None:
log.warning("Expected a public key, but did not receive one")
return defer.fail(Failure(ValueError("Expected but did not receive a public key")))
else:
return defer.succeed({})
+127
View File
@@ -0,0 +1,127 @@
from lbrynet.core.Strategy import get_default_strategy, OnlyFreeStrategy
from lbrynet import conf
from decimal import Decimal
class BasePaymentRateManager(object):
def __init__(self, rate=None, info_rate=None):
self.min_blob_data_payment_rate = rate if rate is not None else conf.settings['data_rate']
self.min_blob_info_payment_rate = (
info_rate if info_rate is not None else conf.settings['min_info_rate'])
class PaymentRateManager(object):
def __init__(self, base, rate=None):
"""
@param base: a BasePaymentRateManager
@param rate: the min blob data payment rate
"""
self.base = base
self.min_blob_data_payment_rate = rate
self.points_paid = 0.0
def get_rate_blob_data(self, peer):
return self.get_effective_min_blob_data_payment_rate()
def accept_rate_blob_data(self, peer, payment_rate):
return payment_rate >= self.get_effective_min_blob_data_payment_rate()
def get_effective_min_blob_data_payment_rate(self):
if self.min_blob_data_payment_rate is None:
return self.base.min_blob_data_payment_rate
return self.min_blob_data_payment_rate
def record_points_paid(self, amount):
self.points_paid += amount
class NegotiatedPaymentRateManager(object):
def __init__(self, base, availability_tracker, generous=None):
"""
@param base: a BasePaymentRateManager
@param availability_tracker: a BlobAvailabilityTracker
@param rate: the min blob data payment rate
"""
self.base = base
self.min_blob_data_payment_rate = base.min_blob_data_payment_rate
self.points_paid = 0.0
self.blob_tracker = availability_tracker
self.generous = generous if generous is not None else conf.settings['is_generous_host']
self.strategy = get_default_strategy(self.blob_tracker,
base_price=self.base.min_blob_data_payment_rate,
is_generous=generous)
def get_rate_blob_data(self, peer, blobs):
response = self.strategy.make_offer(peer, blobs)
return response.rate
def accept_rate_blob_data(self, peer, blobs, offer):
offer = self.strategy.respond_to_offer(offer, peer, blobs)
self.strategy.update_accepted_offers(peer, offer)
return offer.is_accepted
def reply_to_offer(self, peer, blobs, offer):
reply = self.strategy.respond_to_offer(offer, peer, blobs)
self.strategy.update_accepted_offers(peer, reply)
return reply
def get_rate_for_peer(self, peer):
return self.strategy.accepted_offers.get(peer, False)
def record_points_paid(self, amount):
self.points_paid += amount
def record_offer_reply(self, peer, offer):
self.strategy.update_accepted_offers(peer, offer)
def price_limit_reached(self, peer):
if peer in self.strategy.pending_sent_offers:
offer = self.strategy.pending_sent_offers[peer]
return (offer.is_too_low and
round(Decimal.from_float(offer.rate), 5) >= round(self.strategy.max_rate, 5))
return False
class OnlyFreePaymentsManager(object):
def __init__(self, **kwargs):
"""
A payment rate manager that will only ever accept and offer a rate of 0.0,
Used for testing
"""
self.base = BasePaymentRateManager(0.0, 0.0)
self.points_paid = 0.0
self.generous = True
self.strategy = OnlyFreeStrategy()
def get_rate_blob_data(self, peer, blobs):
response = self.strategy.make_offer(peer, blobs)
return response.rate
def accept_rate_blob_data(self, peer, blobs, offer):
offer = self.strategy.respond_to_offer(offer, peer, blobs)
self.strategy.update_accepted_offers(peer, offer)
return offer.is_accepted
def reply_to_offer(self, peer, blobs, offer):
reply = self.strategy.respond_to_offer(offer, peer, blobs)
self.strategy.update_accepted_offers(peer, reply)
return reply
def get_rate_for_peer(self, peer):
return self.strategy.accepted_offers.get(peer, False)
def record_points_paid(self, amount):
self.points_paid += amount
def record_offer_reply(self, peer, offer):
self.strategy.update_accepted_offers(peer, offer)
def price_limit_reached(self, peer):
if peer in self.strategy.pending_sent_offers:
offer = self.strategy.pending_sent_offers[peer]
if offer.rate > 0.0:
return True
return False
+46
View File
@@ -0,0 +1,46 @@
import datetime
from collections import defaultdict
from lbrynet.core import utils
# Do not create this object except through PeerManager
class Peer(object):
def __init__(self, host, port):
self.host = host
self.port = port
# If a peer is reported down, we wait till this time till reattempting connection
self.attempt_connection_at = None
# Number of times this Peer has been reported to be down, resets to 0 when it is up
self.down_count = 0
# Number of succesful connections (with full protocol completion) to this peer
self.success_count = 0
self.score = 0
self.stats = defaultdict(float) # {string stat_type, float count}
def is_available(self):
if self.attempt_connection_at is None or utils.today() > self.attempt_connection_at:
return True
return False
def report_up(self):
self.down_count = 0
self.attempt_connection_at = None
def report_success(self):
self.success_count += 1
def report_down(self):
self.down_count += 1
timeout_time = datetime.timedelta(seconds=60 * self.down_count)
self.attempt_connection_at = utils.today() + timeout_time
def update_score(self, score_change):
self.score += score_change
def update_stats(self, stat_type, count):
self.stats[stat_type] += count
def __str__(self):
return '{}:{}'.format(self.host, self.port)
def __repr__(self):
return 'Peer({!r}, {!r})'.format(self.host, self.port)
+19
View File
@@ -0,0 +1,19 @@
from twisted.internet import defer
class DummyPeerFinder(object):
"""This class finds peers which have announced to the DHT that they have certain blobs"""
def __init__(self):
pass
def run_manage_loop(self):
pass
def stop(self):
pass
def find_peers_for_blob(self, blob_hash):
return defer.succeed([])
def get_most_popular_hashes(self, num_to_return):
return []
+14
View File
@@ -0,0 +1,14 @@
from lbrynet.core.Peer import Peer
class PeerManager(object):
def __init__(self):
self.peers = []
def get_peer(self, host, port):
for peer in self.peers:
if peer.host == host and peer.port == port:
return peer
peer = Peer(host, port)
self.peers.append(peer)
return peer
+62
View File
@@ -0,0 +1,62 @@
from zope.interface import implementer
from decimal import Decimal
from lbrynet.interfaces import IBlobPriceModel
from lbrynet import conf
def get_default_price_model(blob_tracker, base_price, **kwargs):
return MeanAvailabilityWeightedPrice(blob_tracker, base_price, **kwargs)
class ZeroPrice(object):
def __init__(self):
self.base_price = 0.0
def calculate_price(self, blob):
return 0.0
class MeanAvailabilityWeightedPrice(object):
"""Calculate mean-blob-availability and stream-position weighted price for a blob
Attributes:
base_price (float): base price
alpha (float): constant, > 0.0 and <= 1.0, used to more highly
value blobs at the beginning of a stream.
alpha defaults to 1.0, which has a null effect
blob_tracker (BlobAvailabilityTracker): blob availability tracker
"""
implementer(IBlobPriceModel)
def __init__(self, tracker, base_price=None, alpha=1.0):
self.blob_tracker = tracker
base_price = base_price if base_price is not None else conf.settings['data_rate']
self.base_price = Decimal(base_price)
self.alpha = Decimal(alpha)
def calculate_price(self, blob):
mean_availability = self.blob_tracker.last_mean_availability
availability = self.blob_tracker.availability.get(blob, [])
index = 0 # blob.index
availability_mult = self._get_availability_multiplier(mean_availability, availability)
price = self.base_price * availability_mult / self._frontload(index)
return round(price, 5)
def _get_availability_multiplier(self, mean_availability, availability):
return Decimal(max(1, mean_availability) / Decimal(max(1, len(availability))))
def _frontload(self, index):
"""Get front-load multiplier, used to weight prices of blobs in a
stream towards the front of the stream.
At index 0, returns 1.0
As index increases, return value approaches 2.0
@param index: blob position in stream
@return: front-load multiplier
"""
return Decimal(2.0) - (self.alpha ** index)
+156
View File
@@ -0,0 +1,156 @@
import logging
from zope.interface import implements
from lbrynet.interfaces import IRateLimiter
from twisted.internet import task
log = logging.getLogger(__name__)
class DummyRateLimiter(object):
def __init__(self):
self.dl_bytes_this_second = 0
self.ul_bytes_this_second = 0
self.total_dl_bytes = 0
self.total_ul_bytes = 0
self.target_dl = 0
self.target_ul = 0
self.tick_call = None
def start(self):
self.tick_call = task.LoopingCall(self.tick)
self.tick_call.start(1)
def tick(self):
self.dl_bytes_this_second = 0
self.ul_bytes_this_second = 0
def stop(self):
if self.tick_call is not None:
self.tick_call.stop()
self.tick_call = None
def set_dl_limit(self, limit):
pass
def set_ul_limit(self, limit):
pass
def report_dl_bytes(self, num_bytes):
self.dl_bytes_this_second += num_bytes
self.total_dl_bytes += num_bytes
def report_ul_bytes(self, num_bytes):
self.ul_bytes_this_second += num_bytes
self.total_ul_bytes += num_bytes
class RateLimiter(object):
"""This class ensures that upload and download rates don't exceed specified maximums"""
implements(IRateLimiter)
#called by main application
def __init__(self, max_dl_bytes=None, max_ul_bytes=None):
self.max_dl_bytes = max_dl_bytes
self.max_ul_bytes = max_ul_bytes
self.dl_bytes_this_interval = 0
self.ul_bytes_this_interval = 0
self.total_dl_bytes = 0
self.total_ul_bytes = 0
self.tick_call = None
self.tick_interval = 0.1
self.dl_throttled = False
self.ul_throttled = False
self.protocols = []
def start(self):
log.info("Starting rate limiter.")
self.tick_call = task.LoopingCall(self.tick)
self.tick_call.start(self.tick_interval)
def tick(self):
self.dl_bytes_this_interval = 0
self.ul_bytes_this_interval = 0
self.unthrottle_dl()
self.unthrottle_ul()
def stop(self):
log.info("Stopping rate limiter.")
if self.tick_call is not None:
self.tick_call.stop()
self.tick_call = None
def set_dl_limit(self, limit):
self.max_dl_bytes = limit
def set_ul_limit(self, limit):
self.max_ul_bytes = limit
#throttling
def check_dl(self):
need_throttle = (self.max_dl_bytes is not None and
self.dl_bytes_this_interval > self.max_dl_bytes * self.tick_interval)
if need_throttle:
from twisted.internet import reactor
reactor.callLater(0, self.throttle_dl)
def check_ul(self):
need_throttle = (self.max_ul_bytes is not None and
self.ul_bytes_this_interval > self.max_ul_bytes * self.tick_interval)
if need_throttle:
from twisted.internet import reactor
reactor.callLater(0, self.throttle_ul)
def throttle_dl(self):
if self.dl_throttled is False:
for protocol in self.protocols:
protocol.throttle_download()
self.dl_throttled = True
def throttle_ul(self):
if self.ul_throttled is False:
for protocol in self.protocols:
protocol.throttle_upload()
self.ul_throttled = True
def unthrottle_dl(self):
if self.dl_throttled is True:
for protocol in self.protocols:
protocol.unthrottle_download()
self.dl_throttled = False
def unthrottle_ul(self):
if self.ul_throttled is True:
for protocol in self.protocols:
protocol.unthrottle_upload()
self.ul_throttled = False
#called by protocols
def report_dl_bytes(self, num_bytes):
self.dl_bytes_this_interval += num_bytes
self.total_dl_bytes += num_bytes
self.check_dl()
def report_ul_bytes(self, num_bytes):
self.ul_bytes_this_interval += num_bytes
self.total_ul_bytes += num_bytes
self.check_ul()
def register_protocol(self, protocol):
if protocol not in self.protocols:
self.protocols.append(protocol)
if self.dl_throttled is True:
protocol.throttle_download()
if self.ul_throttled is True:
protocol.throttle_upload()
def unregister_protocol(self, protocol):
if protocol in self.protocols:
self.protocols.remove(protocol)
Loaded 100 of 407 files, more files were not shown because too many files have changed in this diff. Show more