Compare commits

..
10 Commits
357 changed files with 18866 additions and 27159 deletions

No files matched your search

+44
View File
@@ -0,0 +1,44 @@
version: 1.0.{build}
environment:
GH_TOKEN:
secure: LiI5jyuHUw6XbH4kC3gP1HX4P/v4rwD/gCNtaFhQu2AvJz1/1wALkp5ECnIxRySN
key_pass:
secure: u6DydPcdrUJlxGL9uc7yQRYG8+5rY6aAEE9nfCSzFyNzZlX9NniOp8Uh5ZKQqX7bGEngLI6ipbLfiJvn0XFnhbn2iTkOuMqOXVJVOehvwlQ=
pfx_key:
secure: 1mwqyRy7hDqDjDK+TIAoaXyXzpNgwruFNA6TPkinUcVM7A+NLD33RQLnfnwVy+R5ovD2pUfhQ6+N0Fqebv6tZh436LIEsock+6IOdpgFwrg=
AWS_ACCESS_KEY_ID:
secure: iVGwoJ7ogspjSmuqr+haVPLglSgQsp6tUZx6mIlKH7Q=
AWS_SECRET_ACCESS_KEY:
secure: zKaqdZGPl0exDL5YhJkb33prSemC9Rzg9S7Lw2wFy1WnJ6ffgl6mQH7jqJDUTqsY
#notifications:
# - provider: Slack
# incoming_webhook:
# secure: LuxwG5OZnnA//gmSXzCKu8/FRqYjCgGfVFqajSsGHeQ1HQNp7rYNhQpsO8/3PK63xKJj3wzt86DJekf9q9Q5OcHa9AHXUQbEveX0psd7elw=
clone_folder: c:\projects\lbry
test_script:
- cd C:\projects\lbry\
- pip install cython
- pip install mock pylint unqlite
- pip install .
- pylint lbrynet
# disable tests for now so that appveyor can build the app
- set PYTHONPATH=.
- trial lbrynet.tests
# TODO: integration tests do not work
#- python -m unittest discover tests/integration
#- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
build_script:
- cd C:\projects\lbry\build\
- ps: .\build.ps1
artifacts:
- path: build\dist\*.zip
name: lbrynet-daemon
-7
View File
@@ -1,7 +0,0 @@
.git
.tox
__pycache__
dist
lbrynet.egg-info
docs
tests
+1 -22
View File
@@ -4,7 +4,7 @@ Thanks for reporting an issue to LBRY and helping us improve!
To make it possible for us to help you, please fill out below information carefully.
Before reporting any issues, please make sure that you're using the latest version.
- App: https://github.com/lbryio/lbry-desktop/releases
- App: https://github.com/lbryio/lbry-app/releases
- Daemon: https://github.com/lbryio/lbry/releases
We are also available on Discord at https://chat.lbry.io
@@ -51,24 +51,3 @@ Tell us what happens instead
## Screenshots
<!-- If a screenshot would help explain the bug, please include one or two here -->
## Internal Use
### Acceptance Criteria
1.
2.
3.
### Definition of Done
- [ ] Tested against acceptance criteria
- [ ] Tested against the assumptions of user story
- [ ] The project builds without errors
- [ ] Unit tests are written and passing
- [ ] Tests on devices/browsers listed in the issue have passed
- [ ] QA performed & issues resolved
- [ ] Refactoring completed
- [ ] Any configuration or build changes documented
- [ ] Documentation updated
- [ ] Peer Code Review performed
-38
View File
@@ -1,38 +0,0 @@
## PR Checklist
Please check all that apply to this PR using "x":
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
- [ ] I have checked that this PR does not introduce a breaking change
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
## PR Type
What kind of change does this PR introduce?
Why is this change necessary?
<!-- Please check all that apply to this PR using "x". -->
- [ ] Bugfix
- [ ] Feature
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
- [ ] Code style update (formatting)
- [ ] Refactoring (no functional changes)
- [ ] Documentation changes
- [ ] Other - Please describe:
## Fixes
Issue Number: N/A
## What is the current behavior?
## What is the new behavior?
## Other information
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
+23 -7
View File
@@ -1,9 +1,25 @@
/build
/dist
/.tox
/.idea
/.coverage
*.pyc
*.egg
*.so
*.xml
*.iml
*.log
*.pem
*.decTest
*.prof
.#*
lbrynet.egg-info
__pycache__
/build/build
/build/dist
/bulid/requirements_base.txt
/lbrynet.egg-info
/docs_build
/lbry-venv
.idea/
.coverage
.DS_Store
# temporary files from the twisted.trial test runner
_trial_temp/
+5 -9
View File
@@ -9,7 +9,7 @@
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS,schema
ignore=CVS
# Add files or directories matching the regex patterns to the
# blacklist. The regex matches against base names, not paths.
@@ -62,7 +62,7 @@ confidence=
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once).You can also use "--disable=all" to
# disable everything first and then re-enable specific checks. For example, if
# disable everything first and then reenable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
@@ -121,11 +121,7 @@ disable=
unidiomatic-typecheck,
global-at-module-level,
inconsistent-return-statements,
keyword-arg-before-vararg,
assignment-from-no-return,
useless-return,
assignment-from-none,
stop-iteration-return
keyword-arg-before-vararg
[REPORTS]
@@ -390,7 +386,7 @@ int-import-graph=
[DESIGN]
# Maximum number of arguments for function / method
max-args=10
max-args=5
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
@@ -409,7 +405,7 @@ max-branches=12
max-statements=50
# Maximum number of parents for a class (see R0901).
max-parents=8
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=7
+39 -122
View File
@@ -1,130 +1,47 @@
sudo: required
dist: xenial
os: linux
dist: trusty
language: python
python: "3.7"
python: 2.7
jobs:
include:
branches:
except:
- gh-pages
- stage: code quality
name: "pylint lbrynet"
install:
- pip install astroid==2.0.4 aiohttp==3.4.4
# newer astroid and aiohttp fails in pylint so we pre-install older version
- pip install pylint
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .
script: pylint lbrynet
- &tests
stage: test
name: "Unit Tests"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .[test]
script:
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.schema tests.unit.wallet tests.unit.components tests.unit.test_conf
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.test_cli
#- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics
after_success:
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- <<: *tests
name: "DHT Tests"
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
- name: "Integration Tests"
install:
- pip install tox-travis coverage
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
script: tox
after_success:
- coverage combine tests/
- bash <(curl -s https://codecov.io/bash)
- stage: build
name: "Windows"
language: generic
services:
- docker
install:
- docker pull lbry/pyinstaller34_32bits:py371
script:
- python scripts/set_build.py
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
file: dist/lbrynet-windows.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
paths:
- lbrynet-windows.zip
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- &build
name: "Linux"
env: OS=linux
install:
- pip3 install pyinstaller
- pip3 install git+https://github.com/lbryio/torba.git
- python scripts/set_build.py
- pip3 install -e .
script:
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
- chmod +x dist/lbrynet
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
- ./dist/lbrynet --version
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
file: dist/lbrynet-${OS}.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
paths:
- lbrynet-${OS}.zip
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- <<: *build
name: "Mac"
os: osx
osx_image: xcode8.3
language: generic
env: OS=mac
cache: false
before_install:
- brew upgrade python || true
- brew upgrade python || true
install:
- python3 --version
- pip3 --version
- pip3 install pyinstaller
- git clone https://github.com/lbryio/torba.git --depth 1
- sed -i -e "s/'plyvel',//" torba/setup.py
- cd torba && pip3 install -e . && cd ..
- python3 scripts/set_build.py
- pip3 install -e .
notifications:
slack:
secure: "Am13HPtpgCMljh0MDVuoFHvQXB8yhf4Kvf/qAeSp5N0vsHGL70CSF9Ahccw8dVPE6mbuak1OGtSUb6/UaErLHkpz3ztaRLkDa9x7CmBB3Kynnh8oO2VbB7b/2ROULqkhF4VZmAnNfwrQrbC3gs8Sybp261Nyc7y4ww15xDYBrk2fyq4ds2DCaJdRxfJUJFonrZ6KXr3fVaXosO6cjuyS8eRodcmrqsT4cCtinjNTD1hGWoH107E4ObSmpVelxQO193KhNJMRiLlEcVkvYUOqIWBtwdGHbNE/6Yeuq1TXgKJ0KeJWAmW3wTfUYNngGXNAsyCnrhul5TKNevNzfIAQZHvRsczYiWPJV6LtohHT0CcUiCXJtvEPOyahEBfwK3etY/xxFqny7N9OEmpdW2sgsEPNPX2LJynJti2rQA9SuAD1ogR3ZpDy/NXoaAZf8PTdPcuNUMULV9PGG7tVrLBecO/W1qO6hdFxwlLdgqGLxAENZgGp++v/DhPk/WvtmHj7iTbRq0nxaTWyX5uKOn2ADH+k/yfutjv6BsQU9xNyPeZEEtuEpc6X6waiYn/8G9vl9PecvKC5H0MgsZ6asAxmg7mZ3VSMFG7mo8ENeOhSZ0Oz6ZTBILL3wFccZA9uJIq7NWmqC9dRiGiuKXBB62No7sINoHg3114e2xYa9qvNmGg="
cache:
directories:
- $HOME/.cache/pip
- $HOME/Library/Caches/pip
- $TRAVIS_BUILD_DIR/.tox
- $TRAVIS_BUILD_DIR/cache/wheel
addons:
#srcclr:
# debug: true
apt:
packages:
- libgmp3-dev
- build-essential
- git
- libssl-dev
- libffi-dev
before_install:
- virtualenv venv
- source venv/bin/activate
install:
- pip install -U pip
- pip install -r requirements.txt
- pip install .
script:
- pip install cython
- pip install mock pylint unqlite
- pylint lbrynet
- PYTHONPATH=. trial lbrynet.tests
- python -m unittest discover lbrynet/tests/integration -v
- rvm install ruby-2.3.1
- rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger
+21 -304
View File
@@ -4,323 +4,40 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/) with
regard to the json-rpc api. As we're currently pre-1.0 release, we
can and probably will change functionality and break backwards compatibility
can and probably will change functionality and break backwards compatability
at anytime.
## [0.30.1] - 2018-12-09
### API
* add `aioupnp_version` to `upnp` field in the response to `status`
* fixed uncaught claim signature errors
* `--blocking` support for `abandon` command (https://github.com/lbryio/lbry/pull/1618)
* don't create SSL keys if SSL is disabled (https://github.com/lbryio/lbry/pull/1644)
* added `account_send` command (https://github.com/lbryio/lbry/pull/1654)
* fixes for claim send to address (https://github.com/lbryio/lbry/pull/1651)
* better handle errors on resolve (https://github.com/lbryio/lbry/pull/1662)
* fix `transaction_show` (https://github.com/lbryio/lbry/pull/1689)
### Schema
* merged `lbryschema` repository into `lbrynet` (https://github.com/lbryio/lbry/pull/1444)
* updated `protobuf` to `3.2.0` (https://github.com/lbryio/lbry/issues/1602)
* use `types` repo for generating python protobufs (https://github.com/lbryio/lbry/pull/1605)
### Wallet
* fixed resolving names with invalid channel signatures
* fixed wallet sync issues
* fixed wallet locking/encryption issues
### DHT
* fixed retrying kademlia rpc calls (https://github.com/lbryio/lbry/pull/1495)
* fixed `KeyError` in `iterativeFindValue`, this is used by `peer_list` and when finding peers to download blobs from (https://github.com/lbryio/lbry/issues/1638)
* settings external ip address and ability to disable more DHT components (https://github.com/lbryio/lbry/pull/1604)
* improved routing table bucket initialization (https://github.com/lbryio/lbry/pull/1633)
### Database
* resolved too many variables in sqlite issue (https://github.com/lbryio/lbry/pull/1612)
### Misc
* switch to Python 3.7
* brought back icon for windows binary
* using custom list of mime types with more types (https://github.com/lbryio/lbry/pull/1700)
* add balance to startup report (https://github.com/lbryio/lbry/pull/1613)
* drop `six` dependency (https://github.com/lbryio/lbry/pull/1660)
* drop `mock` dependency (https://github.com/lbryio/lbry/pull/1661)
## [0.30.0] - 2018-10-22
This release is the most ambitious upgrade to lbrynet yet; including a brand
new wallet implementation, switch to Python 3 of the entire code base and
many changes to the API such as pagination and a more consistent output from
most commands.
## [Unreleased]
### Security
* upgraded `cryptography` package.
* switch to `coincurve` for private / public key generation (addresses)
* support for connecting to daemon via SSL
* improved workflow for encrypt/decrypt of wallet
*
*
### API
* unified all command line executables into a single `lbrynet` executable.
* added `use_https` configuration setting (off by default), if true the api
server will generate a self signed ssl certificate when started and only
handle api requests on the https address (default of `https://localhost:5279/lbryapi`)
* deprecated `daemon_stop` command, use `stop` instead.
* deprecated `wallet_balance` command, use `account_balance` instead.
* deprecated `wallet_unlock` command, use `account_unlock` instead.
* deprecated `wallet_decrypt` command, use `account_decrypt` instead.
* deprecated `wallet_encrypt` command, use `account_encrypt` instead.
* deprecated `wallet_prefill_addresses` command, use `account_fund` instead.
* deprecated `wallet_list` command, use `address_list` instead.
* deprecated `wallet_is_address_mine` command, use `address_is_mine` instead.
* deprecated `wallet_public_key` command.
* deprecated `wallet_new_address` command.
* deprecated `wallet_unused_address` command, use `address_unused` instead.
* deprecated `claim_renew` command.
* added `account_list` command to list accounts including their balance.
* added `account_add` command to add a previously created account from seed or private key.
* added `account_create` command to generate a new account.
* added `account_remove` command to remove an account from wallet.
* added `account_set` command to change a setting on an account.
* added `account_balance` command to get just the account balance.
* added `account_unlock` command to unlock an encrypted account.
* added `account_lock` command to lock an encrypted account.
* added `account_encrypt` command to encrypt an account.
* added `account_decrypt` command to decrypt an account.
* added `account_fund` command to move funds between or within an account in various ways.
* added `account_max_address_gap` command to find large gaps of unused addresses.
* added `address_list` command to list addresses.
* added `address_is_mine` command to check if an address is one of your addresses.
* added `address_unused` command to get existing or generate a new unused address.
* added pagination support for `address_list`, `channel_list`, `claim_list_mine`,
`transaction_list` and `utxo_list`.
* added `upnp` field to `status` response
* removed `send_amount_to_address` command previously marked as deprecated
* removed `channel_list_mine` command previously marked as deprecated
* removed `get_availability` command previously marked as deprecated
* updated `txupnp` dependency to `aioupnp` (UPnP for asyncio)
### Wallet
* changed to a new wallet implementation: [torba](https://github.com/lbryio/torba).
* changed wallet file format to support multiple accounts in one wallet.
* moved transaction data from wallet file into an sqlite database.
* changed channel certificates to be keyed by txid:nout instead of claim_id which
makes it possible to recover old certificates.
### File Manager
* Extensive internal changes as a result of porting to Python 3.
### DHT
* Extensive internal changes as a result of porting to Python 3.
### P2P
* Extensive internal changes as a result of porting to Python 3.
### Reflector
* No major changes to reflector.
### Database
* No changes to existing storage module.
### Known Issues
* Channel certificate import/export is currently not working.
## [0.21.2] - 2018-08-23
### Fixed
* issue in dht ping queue where enqueued pings that aren't yet due wouldn't be rescheduled
* blob mirror downloader not finishing streams that were partially uploaded at the time of the download attempt (https://github.com/lbryio/lbry/issues/1376)
## [0.21.1] - 2018-08-13
### Fixed
* `download_progress` field in `blockchain_headers` section of `status` not initializing correctly when resuming a download (https://github.com/lbryio/lbry/issues/1355)
* `wallet_send` not accepting decimal amounts (https://github.com/lbryio/lbry/issues/1356 commit https://github.com/lbryio/lbry/commit/1098ca0494ece420c70fc57f69d6d388715a99b8)
### Added
* `is_locked` to `wallet` in `status` response (https://github.com/lbryio/lbry/issues/1354, commit https://github.com/lbryio/lbry/commit/153022a1a7122ab6d31d3db433dccbe469bbcb3c)
### Changed
* Bumped `lbryum` requirement to 3.2.4 [see changelog](https://github.com/lbryio/lbryum/blob/master/CHANGELOG.md#324---2018-08-13)
## [0.21.0] - 2018-08-09
### Fixed
* check `claim_address` and `change_address` earlier on publishing, to avoid hard to understand errors later in the process (pr https://github.com/lbryio/lbry/pull/1347)
* loggly error reporting not following `share_usage_data` (pr https://github.com/lbryio/lbry/pull/1328)
* improper error handling when data is not valid JSON (pr https://github.com/lbryio/lbry/pull/1326)
* blob mirroring being set in unrelated tests, making them fail (pr https://github.com/lbryio/lbry/pull/1348)
* http blob mirroring edge cases (pr https://github.com/lbryio/lbry/pull/1315)
* external ports in log messages not showing the correct external port from the upnp redirects (https://github.com/lbryio/lbry/issues/1338) (pr https://github.com/lbryio/lbry/pull/1349)
* miniupnpc fallback issues in txupnp (https://github.com/lbryio/lbry/issues/1341) (pr https://github.com/lbryio/lbry/pull/1349)
* upnp error when disabled on router and a non-gateway is found, such as chromecast (https://github.com/lbryio/lbry/issues/1352) (https://github.com/lbryio/lbry/commit/dca4af942fbe95547794213775f0a62cd04a393f)
*
*
### Deprecated
* automatic claim renew, this is no longer needed
*
*
### Changed
* api server class to use components, and for all JSONRPC API commands to be callable so long as the required components are available. (pr https://github.com/lbryio/lbry/pull/1294)
* return error messages when required conditions on components are not met for API calls (pr https://github.com/lbryio/lbry/pull/1328)
* `status` to no longer return a base58 encoded `lbry_id`, instead return this as the hex encoded `node_id` in a new `dht` field. (pr https://github.com/lbryio/lbry/pull/1328)
* `startup_status` field in the response to `status` to be a dict of component names to status booleans (pr https://github.com/lbryio/lbry/pull/1328)
* renamed the `blockchain_status` field in the response to `status` to `wallet` (pr https://github.com/lbryio/lbry/pull/1328)
* moved and renamed `wallet_is_encrypted` to `is_encrypted` in the `wallet` field in the response to `status` (pr https://github.com/lbryio/lbry/pull/1328)
* moved wallet, upnp and dht startup code from `Session` to `Components` (pr https://github.com/lbryio/lbry/pull/1328)
* attempt blob downloads from http mirror sources (by default) concurrently to p2p sources (pr https://github.com/lbryio/lbry/pull/1233)
* replace miniupnpc with [txupnp](https://github.com/lbryio/txupnp). Since txupnp is still under development, it will internally fall back to miniupnpc. (pr https://github.com/lbryio/lbry/pull/1328)
* simplified test_misc.py in the functional tests (pr https://github.com/lbryio/lbry/pull/1328)
* update `cryptography` requirement to 2.3 (pr https://github.com/lbryio/lbry/pull/1333)
*
*
### Added
* `skipped_components` list to the response from `status` (pr https://github.com/lbryio/lbry/pull/1328)
* component statuses (`blockchain_headers`, `dht`, `wallet`, `blob_manager` `hash_announcer`, and `file_manager`) to the response to `status` (pr https://github.com/lbryio/lbry/pull/1328)
* `skipped_components` config setting, accepts a list of names of components to not run (pr https://github.com/lbryio/lbry/pull/1294)
* `ComponentManager` for managing the life-cycles of dependencies (pr https://github.com/lbryio/lbry/pull/1294)
* `requires` decorator to register the components required by a `jsonrpc_` command, to facilitate commands registering asynchronously (pr https://github.com/lbryio/lbry/pull/1294)
* unit tests for `ComponentManager` (pr https://github.com/lbryio/lbry/pull/1294)
* script to generate docs/api.json file (https://github.com/lbryio/lbry.tech/issues/42)
* additional information to the balance error message when editing a claim (pr https://github.com/lbryio/lbry/pull/1309)
* `address` and `port` arguments to `peer_ping` (https://github.com/lbryio/lbry/issues/1313) (pr https://github.com/lbryio/lbry/pull/1299)
* ability to download from HTTP mirrors by setting `download_mirrors` (prs https://github.com/lbryio/lbry/pull/1233 and https://github.com/lbryio/lbry/pull/1315)
* ability to filter peers from an iterative find value operation (finding peers for a blob). This is used to filter peers we've already found for a blob when accumulating the list of peers. (pr https://github.com/lbryio/lbry/pull/1287)
*
*
### Removed
* `session_status` argument and response field from `status` (pr https://github.com/lbryio/lbry/pull/1328)
* most of the internal attributes from `Daemon` (pr https://github.com/lbryio/lbry/pull/1294)
## [0.20.4] - 2018-07-18
### Fixed
* spelling errors in messages printed by `lbrynet-cli`
* high CPU usage when a stream is incomplete and the peers we're requesting from have no more blobs to send us (https://github.com/lbryio/lbry/pull/1301)
### Changed
* keep track of failures for DHT peers for up to ten minutes instead of indefinitely (https://github.com/lbryio/lbry/pull/1300)
* skip ignored peers from iterative lookups instead of blocking the peer who returned them to us too (https://github.com/lbryio/lbry/pull/1300)
* if a node becomes ignored during an iterative find cycle remove it from the shortlist so that we can't return it as a result nor try to probe it anyway (https://github.com/lbryio/lbry/pull/1303)
## [0.20.3] - 2018-07-03
### Fixed
* `blob_list` raising an error when blobs in a stream haven't yet been created (8a0d0b44ddf9cbeb2a9074eed39d6064ce21df64)
* stopping a download potentially raising an attribute error (https://github.com/lbryio/lbry/pull/1269)
* file manager startup locking up when there are many files for some channels (https://github.com/lbryio/lbry/pull/1281)
* improper sorting when getting the closest peers to a hash (https://github.com/lbryio/lbry/pull/1282)
### Changed
* raised the default `peer_search_timeout` setting from 3 to 30 and added logging for when it happens (https://github.com/lbryio/lbry/pull/1283)
* change iterative find stop condition on find value to allow it to continue until a value is found or it times out (https://github.com/lbryio/lbry/pull/1283)
* include all of our own blobs in the local dht datastore (as if we had announced them to ourselves) (https://github.com/lbryio/lbry/pull/1280)
* ignore dht `store` token validation errors for the first expiration-time after startup (fixes failed `store` requests after a restart) (https://github.com/lbryio/lbry/pull/1280)
### Removed
* `jsonrpclib` as a requirement for the project (https://github.com/lbryio/lbry/pull/1274)
## [0.20.2] - 2018-06-23
### Changed
* Bumped `lbryschema` requirement to 0.0.16 [see changelog](https://github.com/lbryio/lbryschema/blob/master/CHANGELOG.md#0016---2018-06-23)
* Bumped `lbryum` requirement to 3.2.3 [see changelog](https://github.com/lbryio/lbryum/blob/master/CHANGELOG.md#323---2018-06-23)
* save claims to sqlite in batches to speed up `resolve` queries for many uris
## [0.20.1] - 2018-06-19
### Fixed
* fixed token validation error when the dht node has just been started (https://github.com/lbryio/lbry/issues/1248)
* fixed a race condition when inserting a blob into the database (https://github.com/lbryio/lbry/issues/1129)
* reflector server incorrectly responding as if it has all the blobs for a stream that was only partially uploaded to it
* `publish` raising a database error when updating a claim that we don't have a file for (https://github.com/lbryio/lbry/issues/1165)
* blob client protocol not tearing itself down properly after a failure (https://github.com/lbryio/lbry/issues/950)
* lockup in wallet startup when one or more lbryumx servers are unavailable (https://github.com/lbryio/lbry/issues/1245)
* download being stopped if the sd blob downloaded and data did not start within the timeout (https://github.com/lbryio/lbry/issues/1172)
### Changed
* Bumped `lbryum` requirement to 3.2.2 [see changelog](https://github.com/lbryio/lbryum/blob/master/CHANGELOG.md#322---2018-06-19)
* `publish` to accept bid as a decimal string
## [0.20.0] - 2018-06-13
### TL;DR
This milestone release includes a large number of bug fixes, changes and additions covering all aspects of the daemon. Notable improvements include:
* Faster and more reliable downloading and uploading of content resulting from substantial work done on the Distributed Hash Table algorithms and protocol.
* Faster blockchain synchronization (headers) by downloading them from Amazon S3 under certain conditions.
* Faster overall app startup due to better optimized SQL queries.
* Power users of the `lbrynet-cli` will benefit from many bug fixes to commonly used commands and improvements in sorting of output.
* Please review the full change log for more details on specific fixes, changes and additions.
### Fixed
* fix payment rate manager typo ([1236](https://github.com/lbryio/lbry/pull/1236))
* handling error from dht clients with old `ping` method
* blobs not being re-announced if no peers successfully stored, now failed announcements are re-queued
* issue where an `AuthAPIClient` (used by `lbrynet-cli`) would fail to update its session secret and keep making new auth sessions, with every other request failing
* `use_auth_http` in a config file being overridden by the default command line argument to `lbrynet-daemon`, now the command line value will only override the config file value if it is provided
* `lbrynet-cli` not automatically switching to the authenticated client if the server is detected to be using authentication. This resulted in `lbrynet-cli` failing to run when `lbrynet-daemon` was run with the `--http-auth` flag
* fixed error when using `claim_show` with `txid` and `nout` arguments
* fixed error when saving server list to conf file ([1209](https://github.com/lbryio/lbry/pull/1209))
### Changed
* if the `use_authentication` setting is configured, use authentication for all api methods instead of only those with the `auth_required` decorator
* regenerate api keys on startup if the using authentication
* support both positional and keyword args for api calls
* `blob_announce` to queue the blob announcement but not block on it
* `peer_list` to return a list of dictionaries instead of a list of lists, added peer node ids to the results
* predictable result sorting for `claim_list` and `claim_list_mine` ([1216](https://github.com/lbryio/lbry/pull/1216) and [1208](https://github.com/lbryio/lbry/pull/1208))
* increase the default `auto_re_reflect_interval` setting to a day and the default `concurrent_announcers` setting to 10
* download blockchain headers from s3 before starting the wallet when the local height is more than `s3_headers_depth` (a config setting) blocks behind ([1177](https://github.com/lbryio/lbry/pull/1177))
* check headers file integrity on startup, removing/truncating the file to force re-download when necessary
* support partial headers file download from S3 ([1189](https://github.com/lbryio/lbry/pull/1189))
* refactor `add_completed_blobs` on storage.py, simplifying into less queries ([1226](https://github.com/lbryio/lbry/pull/1226))
* full verification of streams only during database migration instead of every startup ([1195](https://github.com/lbryio/lbry/pull/1195))
* database batching functions for starting up the file manager
* added `single_announce` and `last_announced_time` columns to the `blob` table in sqlite
* track successful reflector uploads in sqlite to minimize how many streams are attempted by auto re-reflect ([1194](https://github.com/lbryio/lbry/pull/1194))
* pass the sd hash to reflector ClientFactory instead of looking it up from the database
* dht logging to be more verbose with errors and warnings
* `store` kademlia rpc method to block on the call finishing and to return storing peer information
* kademlia protocol to minimally delay writes to the UDP socket
* several internal dht functions to use inlineCallbacks
* `DHTHashAnnouncer` and `Node` manage functions to use `LoopingCall`s instead of scheduling with `callLater`.
* refactored `DHTHashAnnouncer` to no longer use locks, use a `DeferredSemaphore` to limit concurrent announcers
* decoupled `DiskBlobManager` from `DHTHashAnnouncer`, get blob hashes to announce from `SQLiteStorage`. The blob manager no longer announces blobs after they are completed, the hash announcer takes care of this now.
* changed the bucket splitting condition in the dht routing table to be more aggressive
* ping dht nodes who have stored to us periodically to determine whether we should include them as an active peer for the hash when we are queried. Nodes that are known to be not reachable by the node storing the record are no longer returned as peers by the storing node.
* changed dht bootstrap join process to better populate the routing table initially
* cache dht node tokens used during announcement to minimize the number of requests that are needed
* implement BEP0005 dht rules to classify nodes as good, bad, or unknown and for when to add them to the routing table (http://www.bittorrent.org/beps/bep_0005.html)
* refactored internal dht contact class to track failure counts/times, the time the contact last replied to us, and the time the node last requested something fom us ([1211](https://github.com/lbryio/lbry/pull/1211))
* refactored dht iterativeFind
* sort dht contacts returned by `findCloseNodes` in the routing table
* `reactor` and `callLater`, `listenUDP`, and `resolve` functions to be configurable (to allow easier testing)
* calls to get the current time to use `reactor.seconds` (to control callLater and LoopingCall timing in tests)
* temporarily disabled data price negotiation, treat all data as free
* disabled Cryptonator price feed
* use `treq` instead of `txrequests` ([1191](https://github.com/lbryio/lbry/pull/1191))
* updated `cryptography` version to 2.2.2
* removed `pycrypto` dependency, replacing all calls to `cryptography`
### Added
* `peer_ping` command
* `--sort` option in `file_list` ([1174](https://github.com/lbryio/lbry/pull/1174))
* `port` field to contacts returned by `routing_table_get`
* configurable `concurrent_announcers` and `s3_headers_depth` settings
* virtual kademlia network and mock udp transport for dht integration tests
* functional tests for bootstrapping the dht, announcing and expiring hashes, finding and pinging nodes, protocol version 0/1 backwards/forwards compatibility, and rejoining the network
* linux distro and desktop name added to analytics ([1218](https://github.com/lbryio/lbry/pull/1218))
* certifi module for Twisted SSL verification on Windows ([1213](https://github.com/lbryio/lbry/pull/1213))
* protocol version to dht requests and to the response from `findValue`
### Removed
* `announce_all` argument from `blob_announce`
* old `blob_announce_all` command
* unused `--wallet` argument to `lbrynet-daemon`, which used to be to support `PTCWallet`.
* `AuthJSONRPCServer.auth_required` decorator ([1161](https://github.com/lbryio/lbry/pull/1161))
* `OptimizedTreeRoutingTable` class used by the dht node for the time being
*
*
## [0.19.3] - 2018-05-04
### Changed
* download blockchain headers from s3 before starting the wallet when the local height is more than s3_headers_depth (a config setting) blocks behind (https://github.com/lbryio/lbry/pull/1177)
* un-deprecated report_bug command (https://github.com/lbryio/lbry/commit/f8e418fb4448a3ed1531657f8b3c608fb568af85)
* download blockchain headers from s3 before starting the wallet when the local height is more than s3_headers_depth (a config setting) blocks behind
* un-deprecated report_bug command
## [0.19.2] - 2018-03-28
### Fixed
@@ -486,7 +203,7 @@ most commands.
* Fixed https://github.com/lbryio/lbry/issues/923
* Fixed concurrent reflects opening too many files
* Fixed cases when reflecting would fail on error conditions
* Fixed deadlocks from occurring during blob writes
* Fixed deadlocks from occuring during blob writes
* Fixed and updated`lbrynet.tests.dht`
* Fixed redundant dht id
* Fixed dht `ping` method
@@ -873,7 +590,7 @@ most commands.
* Removed check_pending logic from Daemon
* Switched to txrequests so requests can use twisted event loop
* Renamed API command file_seed to file_set_status
* Don't add expected payment to wallet when payment rate is 0
* Dont add expected payment to wallet when payment rate is 0
### Fixed
* Fix restart procedure in DaemonControl
* Create download directory if it doesn't exist
@@ -904,7 +621,7 @@ most commands.
* Added string comparison to ClaimOutpoint (needed to look things up by outpoint)
* Remove unused API commands from daemon
* Fix file filter `outpoint`
* Made dictionary key names in API command outputs to be more consistent
* Made dictionary key names in API commmand outputs to be more consistent
### Added
* Add file filters: `claim_id`, `outpoint`, and `rowid`
* Make loggly logs less verbose
@@ -956,7 +673,7 @@ most commands.
* Include download url in version check
### Fixed
* add missing traceback to logging
* add misssing traceback to logging
## [0.8.3] - 2017-02-15
### Fixed
+65 -36
View File
@@ -1,48 +1,70 @@
# Installing LBRY
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
The recommended way to install LBRY is to use an installer. We provide installers for all major operating systems. See the [README](README.md).
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
Here's a video walkthrough of this setup, which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch):
[![Setup for development](https://spee.ch/2018-10-04-17-13-54-017046806.png)](https://spee.ch/967f99344308f1e90f0620d91b6c93e4dfb240e0/lbrynet-dev-setup.mp4)
These instructions are for installing from source code, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
## Prerequisites
Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred). Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
Before starting, you'll need to install some additional libraries depending on your operating system.
After installing python 3, you'll need to install some additional libraries depending on your operating system.
### Python
### Virtualenv
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
### Windows
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
LBRY requires Python 2.7. Python 3 is not supported.
### macOS
### OS X Prerequisites
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
Install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
Remaining dependencies can then be installed by running:
```
brew install python3 protobuf
brew install mpfr libmpc protobuf
sudo easy_install pip
sudo pip install virtualenv
```
### Linux
### Linux Prerequisites
On Ubuntu (we recommend 18.04), install the following:
On Ubuntu (we recommend 16.04), install the following:
```
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
sudo apt-get install build-essential python2.7 python2.7-dev python-pip git python-virtualenv libssl-dev libffi-dev python-protobuf
```
On Raspbian, you will also need to install `python-pyparsing`.
If you're running another Linux distro, install the equivalent of the above packages for your system.
If you're running another Linux flavor, install the equivalent of the above packages for your system.
### Windows Prerequisites
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
Add C:\MinGW\bin to the windows PATH.
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing:
```
[build]
compiler = mingw32
```
If using virtualenv, which is recommended, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
Currently, available binaries include:
- Cython
- Twisted
- Zope.interface
- pywin32
- requests
Install the above binaries with `pip install *.whl`
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
Finally, you'll need [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9). Download the source and compile with MinGW by running `mingw32make.bat`. Then install the module by running `python setupmingw32.py install`.
## Installation
@@ -51,27 +73,34 @@ To install:
```
git clone https://github.com/lbryio/lbry.git
cd lbry
virtualenv lbry-venv --python=python3.7
virtualenv lbry-venv
source lbry-venv/bin/activate
python --version # Python 2 is not supported. Make sure you're on Python 3.7
pip install -e .
python --version # Python 3 is not supported. Make sure you're on Python 2.7
pip install -U -r requirements.txt
pip install --editable .
```
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
To start LBRY, run `lbrynet-daemon` in a terminal.
## Run the tests
To run the unit tests from the repo directory:
```
trial --reactor=asyncio tests.unit
```
To verify your install, `which lbrynet-daemon` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
## Usage
## Making a Change
To start the API server:
`lbrynet start`
If you are not already inside the virtual environment, run:
```
source lbry-venv/bin/activate
```
Then run:
```
pip install --editable .
```
This will install `lbrynet-daemon` in such a way that the changes you make to the code will be used as soon as you restart the daemon.
Happy hacking!
+34 -38
View File
@@ -1,56 +1,52 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
[![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry)
[![Coverage Status](https://coveralls.io/repos/github/lbryio/lbry/badge.svg)](https://coveralls.io/github/lbryio/lbry)
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
# LBRY
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized network.
* Built on Python 3.7+ and `asyncio`.
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
This repo is a reference implementation of the [LBRY API](https://lbry.io/api).
## Installation
It provides a daemon that can interact with the network via a json-rpc interface over HTTP.
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
## Installing
## Usage
**Note**: This project no longer bundles a graphic interface (browser). If you want to use LBRY via a GUI, [use the LBRY App](https://github.com/lbryio/lbry-app).
Run `lbrynet start` to launch the API server.
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases, for macOS, Debian-based Linux, and Windows.
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
Installing from source is also relatively painless, full instructions are in [INSTALL.md](INSTALL.md)
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
## Running
With the daemon running, `lbrynet commands` will show you a list of commands.
Run `lbrynet-daemon` to launch the daemon.
The full API is documented [here](https://lbry.tech/api/sdk).
## Using
## Running from source
By default, `lbrynet-daemon` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
Our [quickstart guide](http://lbry.io/quickstart) provides a simple walkthrough and examples for learning.
## Contributing
The full API is documented [here](https://lbryio.github.io/lbry/cli).
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.io/faq/contributing) link.
## What is LBRY?
LBRY is a protocol that provides a fully decentralized network for the discovery, distribution, and payment of data.
It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, and rights and access rules.
LBRY also provides a data network consists of peers uploading and downloading data from other peers, possibly in exchange for payments, and a distributed hash table, used by peers to discover other peers.
## Contributions
To contribute, [join us on Discord](https://chat.lbry.io/) or contact jeremy@lbry.io. Pull requests are also welcome.
## Support
Please open an issue and describe your situation in detail. We will respond as soon as we can.
For private issues, contact jeremy@lbry.io.
## License
This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
## Contact
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
## Additional information and links
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry/blob/master/example_daemon_settings.yml).
See [LICENSE](LICENSE)
+33
View File
@@ -0,0 +1,33 @@
$env:Path += ";C:\MinGW\bin\"
$env:Path += ";C:\Program Files (x86)\Windows Kits\10\bin\x86\"
gcc --version
mingw32-make --version
# build/install miniupnpc manually
tar zxf miniupnpc-1.9.tar.gz
cd miniupnpc-1.9
mingw32-make -f Makefile.mingw
python setupmingw32.py build --compiler=mingw32
python setupmingw32.py install
cd ..\
Remove-Item -Recurse -Force miniupnpc-1.9
# copy requirements from lbry, but remove miniupnpc (installed manually)
Get-Content ..\requirements.txt | Select-String -Pattern 'miniupnpc' -NotMatch | Out-File requirements_base.txt
python set_build.py
pip install -r requirements.txt
pip install ..\.
pyinstaller -y daemon.onefile.spec
pyinstaller -y cli.onefile.spec
pyinstaller -y console.onefile.spec
nuget install secure-file -ExcludeVersion
secure-file\tools\secure-file -decrypt .\lbry2.pfx.enc -secret "$env:pfx_key"
signtool.exe sign /f .\lbry2.pfx /p "$env:key_pass" /tr http://tsa.starfieldtech.com /td SHA256 /fd SHA256 dist\*.exe
python zip_daemon.py
python upload_assets.py
Executable
+59
View File
@@ -0,0 +1,59 @@
#!/bin/bash
set -euo pipefail
set -x
ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
cd "$ROOT"
BUILD_DIR="$ROOT/build"
FULL_BUILD="${FULL_BUILD:-false}"
if [ -n "${TEAMCITY_VERSION:-}" -o -n "${APPVEYOR:-}" ]; then
FULL_BUILD="true"
fi
[ -d "$BUILD_DIR/bulid" ] && rm -rf "$BUILD_DIR/build"
[ -d "$BUILD_DIR/dist" ] && rm -rf "$BUILD_DIR/dist"
if [ "$FULL_BUILD" == "true" ]; then
# install dependencies
$BUILD_DIR/prebuild.sh
VENV="$BUILD_DIR/venv"
if [ -d "$VENV" ]; then
rm -rf "$VENV"
fi
virtualenv "$VENV"
set +u
source "$VENV/bin/activate"
set -u
# must set build before installing lbrynet. otherwise it has no effect
python "$BUILD_DIR/set_build.py"
fi
cp "$ROOT/requirements.txt" "$BUILD_DIR/requirements_base.txt"
(
cd "$BUILD_DIR"
pip install -r requirements.txt
)
(
cd "$BUILD_DIR"
pyinstaller -y daemon.onefile.spec
pyinstaller -y cli.onefile.spec
pyinstaller -y console.onefile.spec
)
python "$BUILD_DIR/zip_daemon.py"
if [ "$FULL_BUILD" == "true" ]; then
# electron-build has a publish feature, but I had a hard time getting
# it to reliably work and it also seemed difficult to configure. Not proud of
# this, but it seemed better to write my own.
python "$BUILD_DIR/upload_assets.py"
deactivate
fi
echo 'Build complete.'
+42
View File
@@ -0,0 +1,42 @@
# -*- mode: python -*-
import platform
import os
dir = 'build';
cwd = os.getcwd()
if os.path.basename(cwd) != dir:
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
repo_base = os.path.abspath(os.path.join(cwd, '..'))
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
system = platform.system()
if system == 'Darwin':
icns = os.path.join(repo_base, 'build', 'icon.icns')
elif system == 'Linux':
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
elif system == 'Windows':
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
else:
print 'Warning: System {} has no icons'.format(system)
icns = None
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-cli', pathex=[cwd])
pyz = PYZ(a.pure, a.zipped_data)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
name='lbrynet-cli',
debug=False,
strip=False,
upx=True,
console=True,
icon=icns
)
+50
View File
@@ -0,0 +1,50 @@
# -*- mode: python -*-
import platform
import os
import lbryum
dir = 'build';
cwd = os.getcwd()
if os.path.basename(cwd) != dir:
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
repo_base = os.path.abspath(os.path.join(cwd, '..'))
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
system = platform.system()
if system == 'Darwin':
icns = os.path.join(repo_base, 'build', 'icon.icns')
elif system == 'Linux':
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
elif system == 'Windows':
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
else:
print 'Warning: System {} has no icons'.format(system)
icns = None
datas = [
(os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language + '.txt'), 'lbryum/wordlist')
for language in ('chinese_simplified', 'japanese', 'spanish','english', 'portuguese')
]
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-console', pathex=[cwd], datas=datas)
pyz = PYZ(a.pure, a.zipped_data)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
name='lbrynet-console',
debug=False,
strip=False,
upx=True,
console=True,
icon=icns
)
+50
View File
@@ -0,0 +1,50 @@
# -*- mode: python -*-
import platform
import os
import lbryum
dir = 'build';
cwd = os.getcwd()
if os.path.basename(cwd) != dir:
raise Exception('pyinstaller build needs to be run from the ' + dir + ' directory')
repo_base = os.path.abspath(os.path.join(cwd, '..'))
execfile(os.path.join(cwd, "entrypoint.py")) # ghetto import
system = platform.system()
if system == 'Darwin':
icns = os.path.join(repo_base, 'build', 'icon.icns')
elif system == 'Linux':
icns = os.path.join(repo_base, 'build', 'icons', '256x256.png')
elif system == 'Windows':
icns = os.path.join(repo_base, 'build', 'icons', 'lbry256.ico')
else:
print 'Warning: System {} has no icons'.format(system)
icns = None
datas = [
(os.path.join(os.path.dirname(lbryum.__file__), 'wordlist', language + '.txt'), 'lbryum/wordlist')
for language in ('chinese_simplified', 'japanese', 'spanish','english', 'portuguese')
]
a = Entrypoint('lbrynet', 'console_scripts', 'lbrynet-daemon', pathex=[cwd], datas=datas)
pyz = PYZ(a.pure, a.zipped_data)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
name='lbrynet-daemon',
debug=False,
strip=False,
upx=True,
console=True,
icon=icns
)
+47
View File
@@ -0,0 +1,47 @@
# https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Setuptools-Entry-Point
def Entrypoint(dist, group, name,
scripts=None, pathex=None, binaries=None, datas=None,
hiddenimports=None, hookspath=None, excludes=None, runtime_hooks=None,
cipher=None, win_no_prefer_redirects=False, win_private_assemblies=False):
import pkg_resources
# get toplevel packages of distribution from metadata
def get_toplevel(dist):
distribution = pkg_resources.get_distribution(dist)
if distribution.has_metadata('top_level.txt'):
return list(distribution.get_metadata('top_level.txt').split())
else:
return []
hiddenimports = hiddenimports or []
packages = []
for distribution in hiddenimports:
packages += get_toplevel(distribution)
scripts = scripts or []
pathex = pathex or []
# get the entry point
ep = pkg_resources.get_entry_info(dist, group, name)
# insert path of the egg at the verify front of the search path
pathex = [ep.dist.location] + pathex
# script name must not be a valid module name to avoid name clashes on import
script_path = os.path.join(workpath, name + '-script.py')
print "creating script for entry point", dist, group, name
with open(script_path, 'w') as fh:
fh.write("import {0}\n".format(ep.module_name))
fh.write("{0}.{1}()\n".format(ep.module_name, '.'.join(ep.attrs)))
for package in packages:
fh.write("import {0}\n".format(package))
return Analysis([script_path] + scripts,
pathex=pathex,
binaries=binaries,
datas=datas,
hiddenimports=hiddenimports,
hookspath=hookspath,
excludes=excludes,
runtime_hooks=runtime_hooks,
cipher=cipher,
win_no_prefer_redirects=win_no_prefer_redirects,
win_private_assemblies=win_private_assemblies
)
File renamed without changes.

Before

Width:  |  Height:  |  Size: 7.4 KiB

After

Width:  |  Height:  |  Size: 7.4 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 6.1 KiB

After

Width:  |  Height:  |  Size: 6.1 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 97 KiB

After

Width:  |  Height:  |  Size: 97 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 361 KiB

After

Width:  |  Height:  |  Size: 361 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 5.3 KiB

After

Width:  |  Height:  |  Size: 5.3 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 31 KiB

After

Width:  |  Height:  |  Size: 31 KiB

Binary file not shown.
Binary file not shown.
+81
View File
@@ -0,0 +1,81 @@
#!/bin/bash
set -euo pipefail
set -x
LINUX=false
OSX=false
if [ "$(uname)" == "Darwin" ]; then
OSX=true
elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
LINUX=true
else
echo "Platform detection failed"
exit 1
fi
SUDO=''
if $LINUX && (( $EUID != 0 )); then
SUDO='sudo'
fi
cmd_exists() {
command -v "$1" >/dev/null 2>&1
return $?
}
set +eu
GITUSERNAME=$(git config --global --get user.name)
if [ -z "$GITUSERNAME" ]; then
git config --global user.name "$(whoami)"
fi
GITEMAIL=$(git config --global --get user.email)
if [ -z "$GITEMAIL" ]; then
git config --global user.email "$(whoami)@lbry.io"
fi
set -eu
if $LINUX; then
INSTALL="$SUDO apt-get install --no-install-recommends -y"
$INSTALL build-essential libssl-dev libffi-dev python2.7-dev wget
elif $OSX && ! cmd_exists brew ; then
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
fi
if ! cmd_exists python; then
if $LINUX; then
$INSTALL python2.7
elif $OSX; then
brew install python
fi
fi
PYTHON_VERSION=$(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')
if [ "$PYTHON_VERSION" != "2.7" ]; then
echo "Python 2.7 required"
exit 1
fi
if ! cmd_exists pip; then
if $LINUX; then
$INSTALL python-pip
$SUDO pip install --upgrade pip
else
echo "Pip required"
exit 1
fi
fi
if $LINUX && [ "$(pip list --format=columns | grep setuptools | wc -l)" -ge 1 ]; then
#$INSTALL python-setuptools
$SUDO pip install setuptools
fi
if ! cmd_exists virtualenv; then
$SUDO pip install virtualenv
fi
+11
View File
@@ -0,0 +1,11 @@
# install daemon requirements (created by build script. see build.sh, build.ps1)
-r requirements_base.txt
# install daemon itself. make sure you run `pip install` from this dir. this is how you do relative file paths with pip
file:../.
# install other build requirements
PyInstaller==3.2.1
requests[security]==2.13.0
uritemplate==3.0.0
boto3==1.4.4
+3 -5
View File
@@ -1,4 +1,3 @@
"""Set the build version to be 'dev', 'qa', 'rc', 'release'"""
import os.path
@@ -16,12 +15,11 @@ def main():
def get_build():
try:
tag = subprocess.check_output(['git', 'describe', '--exact-match', '--all']).strip()
if re.match('tags\/v\d+\.\d+\.\d+rc\d+$', tag.decode()):
tag = subprocess.check_output(['git', 'describe', '--exact-match']).strip()
if re.match('v\d+\.\d+\.\d+rc\d+', tag):
return 'rc'
elif re.match('tags\/v\d+\.\d+\.\d+$', tag.decode()):
else:
return 'release'
return 'qa'
except subprocess.CalledProcessError:
# if the build doesn't have a tag
return 'qa'
+143
View File
@@ -0,0 +1,143 @@
import glob
import json
import os
import subprocess
import sys
import github
import uritemplate
import boto3
def main():
upload_to_github_if_tagged('lbryio/lbry')
upload_to_s3('daemon')
def get_asset_filename():
this_dir = os.path.dirname(os.path.realpath(__file__))
return glob.glob(this_dir + '/dist/*.zip')[0]
def upload_to_s3(folder):
tag = subprocess.check_output(['git', 'describe', '--always', '--abbrev=8', 'HEAD']).strip()
commit_date = subprocess.check_output([
'git', 'show', '-s', '--format=%cd', '--date=format:%Y%m%d-%H%I%S', 'HEAD']).strip()
asset_path = get_asset_filename()
bucket = 'releases.lbry.io'
key = folder + '/' + commit_date + '-' + tag + '/' + os.path.basename(asset_path)
print "Uploading " + asset_path + " to s3://" + bucket + '/' + key + ''
if 'AWS_ACCESS_KEY_ID' not in os.environ or 'AWS_SECRET_ACCESS_KEY' not in os.environ:
print 'Must set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to publish assets to s3'
return 1
s3 = boto3.resource(
's3',
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
config=boto3.session.Config(signature_version='s3v4')
)
s3.meta.client.upload_file(asset_path, bucket, key)
def upload_to_github_if_tagged(repo_name):
try:
current_tag = subprocess.check_output(
['git', 'describe', '--exact-match', 'HEAD']).strip()
except subprocess.CalledProcessError:
print 'Not uploading to GitHub as we are not currently on a tag'
return 1
print "Current tag: " + current_tag
if 'GH_TOKEN' not in os.environ:
print 'Must set GH_TOKEN in order to publish assets to a release'
return 1
gh_token = os.environ['GH_TOKEN']
auth = github.Github(gh_token)
repo = auth.get_repo(repo_name)
if not check_repo_has_tag(repo, current_tag):
print 'Tag {} is not in repo {}'.format(current_tag, repo)
# TODO: maybe this should be an error
return 1
asset_path = get_asset_filename()
print "Uploading " + asset_path + " to Github tag " + current_tag
release = get_github_release(repo, current_tag)
upload_asset_to_github(release, asset_path, gh_token)
def check_repo_has_tag(repo, target_tag):
tags = repo.get_tags().get_page(0)
for tag in tags:
if tag.name == target_tag:
return True
return False
def get_github_release(repo, current_tag):
for release in repo.get_releases():
if release.tag_name == current_tag:
return release
raise Exception('No release for {} was found'.format(current_tag))
def upload_asset_to_github(release, asset_to_upload, token):
basename = os.path.basename(asset_to_upload)
for asset in release.raw_data['assets']:
if asset['name'] == basename:
print 'File {} has already been uploaded to {}'.format(basename, release.tag_name)
return
upload_uri = uritemplate.expand(release.upload_url, {'name': basename})
count = 0
while count < 10:
try:
output = _curl_uploader(upload_uri, asset_to_upload, token)
if 'errors' in output:
raise Exception(output)
else:
print 'Successfully uploaded to {}'.format(output['browser_download_url'])
except Exception:
print 'Failed uploading on attempt {}'.format(count + 1)
count += 1
def _curl_uploader(upload_uri, asset_to_upload, token):
# using requests.post fails miserably with SSL EPIPE errors. I spent
# half a day trying to debug before deciding to switch to curl.
#
# TODO: actually set the content type
print 'Using curl to upload {} to {}'.format(asset_to_upload, upload_uri)
cmd = [
'curl',
'-sS',
'-X', 'POST',
'-u', ':{}'.format(os.environ['GH_TOKEN']),
'--header', 'Content-Type: application/octet-stream',
'--data-binary', '@-',
upload_uri
]
# '-d', '{"some_key": "some_value"}',
print 'Calling curl:'
print cmd
print
with open(asset_to_upload, 'rb') as fp:
p = subprocess.Popen(cmd, stdin=fp, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
stdout, stderr = p.communicate()
print 'curl return code:', p.returncode
if stderr:
print 'stderr output from curl:'
print stderr
print 'stdout from curl:'
print stdout
return json.loads(stdout)
if __name__ == '__main__':
sys.exit(main())
+29
View File
@@ -0,0 +1,29 @@
import os
import platform
import subprocess
import sys
import zipfile
def main():
this_dir = os.path.dirname(os.path.realpath(__file__))
tag = subprocess.check_output(['git', 'describe']).strip()
zipfilename = 'lbrynet-daemon-{}-{}.zip'.format(tag, get_system_label())
full_filename = os.path.join(this_dir, 'dist', zipfilename)
executables = ['lbrynet-daemon', 'lbrynet-cli', 'lbrynet-console']
ext = '.exe' if platform.system() == 'Windows' else ''
with zipfile.ZipFile(full_filename, 'w') as myzip:
for executable in executables:
myzip.write(os.path.join(this_dir, 'dist', executable + ext), executable + ext)
def get_system_label():
system = platform.system()
if system == 'Darwin':
return 'macos'
else:
return system.lower()
if __name__ == '__main__':
sys.exit(main())
+3 -3
View File
@@ -100,7 +100,7 @@
<div class="md-flex__cell md-flex__cell--shrink">
<a href="/" title="LBRY" class="md-header-nav__button md-logo">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="24" height="24">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="24" height="24">
</a>
</div>
@@ -196,7 +196,7 @@
<label class="md-nav__title md-nav__title--site" for="drawer">
<span class="md-nav__button md-logo">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" alt="LBRY logo" width="48" height="48">
<img src="https://s3.amazonaws.com/files.lbry.io/logo-square-white-bookonly.png" width="48" height="48">
</span>
LBRY
@@ -304,4 +304,4 @@
</body>
</html>
</html>
-1605
View File
File diff suppressed because it is too large. Load diff
+95 -190
View File
@@ -307,13 +307,6 @@
blob_list
</a>
</li>
<li class="md-nav__item">
<a href="#blob_reflect" title="blob_reflect" class="md-nav__link">
blob_reflect
</a>
</li>
<li class="md-nav__item">
@@ -482,13 +475,6 @@
peer_list
</a>
</li>
<li class="md-nav__item">
<a href="#peer_ping" title="peer_ping" class="md-nav__link">
peer_ping
</a>
</li>
<li class="md-nav__item">
@@ -496,13 +482,6 @@
publish
</a>
</li>
<li class="md-nav__item">
<a href="#report_bug" title="report_bug" class="md-nav__link">
report_bug
</a>
</li>
<li class="md-nav__item">
@@ -730,13 +709,6 @@
blob_list
</a>
</li>
<li class="md-nav__item">
<a href="#blob_reflect" title="blob_reflect" class="md-nav__link">
blob_reflect
</a>
</li>
<li class="md-nav__item">
@@ -905,13 +877,6 @@
peer_list
</a>
</li>
<li class="md-nav__item">
<a href="#peer_ping" title="peer_ping" class="md-nav__link">
peer_ping
</a>
</li>
<li class="md-nav__item">
@@ -919,13 +884,6 @@
publish
</a>
</li>
<li class="md-nav__item">
<a href="#report_bug" title="report_bug" class="md-nav__link">
report_bug
</a>
</li>
<li class="md-nav__item">
@@ -1113,16 +1071,18 @@
<pre><code class="text">Announce blobs to the DHT
Usage:
blob_announce [&lt;blob_hash&gt; | --blob_hash=&lt;blob_hash&gt;]
[&lt;stream_hash&gt; | --stream_hash=&lt;stream_hash&gt;] | [&lt;sd_hash&gt; | --sd_hash=&lt;sd_hash&gt;]
blob_announce [--announce_all] [&lt;blob_hash&gt; | --blob_hash=&lt;blob_hash&gt;]
[&lt;stream_hash&gt; | --stream_hash=&lt;stream_hash&gt;]
[&lt;sd_hash&gt; | --sd_hash=&lt;sd_hash&gt;]
Options:
--blob_hash=&lt;blob_hash&gt; : (str) announce a blob, specified by blob_hash
--stream_hash=&lt;stream_hash&gt; : (str) announce all blobs associated with
stream_hash
--sd_hash=&lt;sd_hash&gt; : (str) announce all blobs associated with
sd_hash and the sd_hash itself
--announce_all=&lt;announce_all&gt; : (bool) announce all the blobs possessed by user
--blob_hash=&lt;blob_hash&gt; : (str) announce a blob, specified by blob_hash
--stream_hash=&lt;stream_hash&gt; : (str) announce all blobs associated with
stream_hash
--sd_hash=&lt;sd_hash&gt; : (str) announce all blobs associated with
sd_hash and the sd_hash itself
Returns:
(bool) true if successful
@@ -1212,20 +1172,6 @@ Returns:
(list) List of blob hashes
</code></pre>
<h2 id="blob_reflect">blob_reflect</h2>
<pre><code class="text">Reflects specified blobs
Usage:
blob_reflect (&lt;blob_hashes&gt;...) [--reflector_server=&lt;reflector_server&gt;]
Options:
--reflector_server=&lt;reflector_server&gt; : (str) reflector address
Returns:
(list) reflected blob hashes
</code></pre>
<h2 id="blob_reflect_all">blob_reflect_all</h2>
<pre><code class="text">Reflects all saved blobs
@@ -1273,7 +1219,8 @@ Returns:
<pre><code class="text">Import serialized channel signing information (to allow signing new claims to the channel)
Usage:
channel_import (&lt;serialized_certificate_info&gt; | --serialized_certificate_info=&lt;serialized_certificate_info&gt;)
channel_import (&lt;serialized_certificate_info&gt; |
--serialized_certificate_info=&lt;serialized_certificate_info&gt;)
Options:
@@ -1454,7 +1401,7 @@ Returns:
'is_spent': (bool) true if claim is abandoned, false otherwise
'name': (str) name of the claim
'permanent_url': (str) permanent url of the claim,
'txid': (str) txid of the claim
'txid': (str) txid of the cliam
'nout': (int) nout of the claim
'value': (str) value of the claim
},
@@ -1579,12 +1526,12 @@ Usage:
Options:
--a_arg : (bool) a arg
--b_arg : (bool) b arg
--pos_arg=&lt;pos_arg&gt; : (int) pos arg
--pos_args=&lt;pos_args&gt; : (int) pos args
--pos_arg2=&lt;pos_arg2&gt; : (int) pos arg 2
--pos_arg3=&lt;pos_arg3&gt; : (int) pos arg 3
--a_arg : a arg
--b_arg : b arg
--pos_arg=&lt;pos_arg&gt; : pos arg
--pos_args=&lt;pos_args&gt; : pos args
--pos_arg2=&lt;pos_arg2&gt; : pos arg 2
--pos_arg3=&lt;pos_arg3&gt; : pos arg 3
Returns:
pos args
@@ -1656,7 +1603,7 @@ Usage:
file_list [--sd_hash=&lt;sd_hash&gt;] [--file_name=&lt;file_name&gt;] [--stream_hash=&lt;stream_hash&gt;]
[--rowid=&lt;rowid&gt;] [--claim_id=&lt;claim_id&gt;] [--outpoint=&lt;outpoint&gt;] [--txid=&lt;txid&gt;] [--nout=&lt;nout&gt;]
[--channel_claim_id=&lt;channel_claim_id&gt;] [--channel_name=&lt;channel_name&gt;]
[--claim_name=&lt;claim_name&gt;] [--full_status] [--sort=&lt;sort_method&gt;...]
[--claim_name=&lt;claim_name&gt;] [--full_status]
Options:
@@ -1674,9 +1621,6 @@ Options:
--claim_name=&lt;claim_name&gt; : (str) get file with matching claim name
--full_status : (bool) full status, populate the
'message' and 'size' fields
--sort=&lt;sort_method&gt; : (str) sort by any property, like 'file_name'
or 'metadata.author'; to specify direction
append ',asc' or ',desc'
Returns:
(list) List of files
@@ -1824,23 +1768,7 @@ Options:
--timeout=&lt;timeout&gt; : (int) peer search timeout in seconds
Returns:
(list) List of contact dictionaries {'host': &lt;peer ip&gt;, 'port': &lt;peer port&gt;, 'node_id': &lt;peer node id&gt;}
</code></pre>
<h2 id="peer_ping">peer_ping</h2>
<pre><code class="text">Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,
if not provided the peer is located first.
Usage:
peer_ping (&lt;node_id&gt; | --node_id=&lt;node_id&gt;) [&lt;address&gt; | --address=&lt;address&gt;] [&lt;port&gt; | --port=&lt;port&gt;]
Options:
--address=&lt;address&gt; : (str) ip address of the peer
--port=&lt;port&gt; : (int) udp port of the peer
Returns:
(str) pong, or {'error': &lt;error message&gt;} if an error is encountered
(list) List of contacts
</code></pre>
<h2 id="publish">publish</h2>
@@ -1870,40 +1798,40 @@ Usage:
Options:
--name=&lt;name&gt; : (str) name of the content
--bid=&lt;bid&gt; : (decimal) amount to back the claim
--metadata=&lt;metadata&gt; : (dict) ClaimDict to associate with the claim.
--file_path=&lt;file_path&gt; : (str) path to file to be associated with name. If provided,
a lbry stream of this file will be used in 'sources'.
If no path is given but a sources dict is provided,
it will be used. If neither are provided, an
error is raised.
--fee=&lt;fee&gt; : (dict) Dictionary representing key fee to download content:
{
'currency': currency_symbol,
'amount': decimal,
'address': str, optional
}
supported currencies: LBC, USD, BTC
If an address is not provided a new one will be
automatically generated. Default fee is zero.
--title=&lt;title&gt; : (str) title of the publication
--description=&lt;description&gt; : (str) description of the publication
--author=&lt;author&gt; : (str) author of the publication
--language=&lt;language&gt; : (str) language of the publication
--license=&lt;license&gt; : (str) publication license
--license_url=&lt;license_url&gt; : (str) publication license url
--thumbnail=&lt;thumbnail&gt; : (str) thumbnail url
--preview=&lt;preview&gt; : (str) preview url
--nsfw=&lt;nsfw&gt; : (bool) whether the content is nsfw
--sources=&lt;sources&gt; : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
--channel_name=&lt;channel_name&gt; : (str) name of the publisher channel name in the wallet
--channel_id=&lt;channel_id&gt; : (str) claim id of the publisher channel, does not check
for channel claim being in the wallet. This allows
publishing to a channel where only the certificate
private key is in the wallet.
--claim_address=&lt;claim_address&gt; : (str) address where the claim is sent to, if not specified
new address will automatically be created
--name=&lt;name&gt; : (str) name of the content
--bid=&lt;bid&gt; : (float) amount to back the claim
--metadata=&lt;metadata&gt; : (dict) ClaimDict to associate with the claim.
--file_path=&lt;file_path&gt; : (str) path to file to be associated with name. If provided,
a lbry stream of this file will be used in 'sources'.
If no path is given but a sources dict is provided,
it will be used. If neither are provided, an
error is raised.
--fee=&lt;fee&gt; : (dict) Dictionary representing key fee to download content:
{
'currency': currency_symbol,
'amount': float,
'address': str, optional
}
supported currencies: LBC, USD, BTC
If an address is not provided a new one will be
automatically generated. Default fee is zero.
--title=&lt;title&gt; : (str) title of the publication
--description=&lt;description&gt; : (str) description of the publication
--author=&lt;author&gt; : (str) author of the publication
--language=&lt;language&gt; : (str) language of the publication
--license=&lt;license&gt; : (str) publication license
--license_url=&lt;license_url&gt; : (str) publication license url
--thumbnail=&lt;thumbnail&gt; : (str) thumbnail url
--preview=&lt;preview&gt; : (str) preview url
--nsfw=&lt;nsfw&gt; : (bool) title of the publication
--sources=&lt;sources&gt; : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
--channel_name=&lt;channel_name&gt; : (str) name of the publisher channel name in the wallet
--channel_id=&lt;channel_id&gt; : (str) claim id of the publisher channel, does not check
for channel claim being in the wallet. This allows
publishing to a channel where only the certificate
private key is in the wallet.
--claim_address=&lt;claim_address&gt; : (str) address where the claim is sent to, if not specified
new address wil automatically be created
Returns:
(dict) Dictionary containing result of the claim
@@ -1911,25 +1839,11 @@ Returns:
'tx' : (str) hex encoded transaction
'txid' : (str) txid of resulting claim
'nout' : (int) nout of the resulting claim
'fee' : (decimal) fee paid for the claim transaction
'fee' : (float) fee paid for the claim transaction
'claim_id' : (str) claim ID of the resulting claim
}
</code></pre>
<h2 id="report_bug">report_bug</h2>
<pre><code class="text">Report a bug to slack
Usage:
report_bug (&lt;message&gt; | --message=&lt;message&gt;)
Options:
--message=&lt;message&gt; : (str) Description of the bug
Returns:
(bool) true if successful
</code></pre>
<h2 id="resolve">resolve</h2>
<pre><code class="text">Resolve given LBRY URIs
@@ -2031,7 +1945,6 @@ Returns:
&lt;bucket index&gt;: [
{
&quot;address&quot;: (str) peer address,
&quot;port&quot;: (int) peer udp port
&quot;node_id&quot;: (str) peer node id,
&quot;blobs&quot;: (list) blob hashes announced by peer
}
@@ -2112,62 +2025,54 @@ Returns:
<pre><code class="text">Get daemon status
Usage:
status
status [--session_status] [--dht_status]
Options:
None
--session_status : (bool) include session status in results
--dht_status : (bool) include dht network and peer status
Returns:
(dict) lbrynet-daemon status
{
'installation_id': (str) installation id - base58,
'is_running': (bool),
'lbry_id': lbry peer id, base58,
'installation_id': installation id, base58,
'is_running': bool,
'is_first_run': bool,
'skipped_components': (list) [names of skipped components (str)],
'startup_status': { Does not include components which have been skipped
'database': (bool),
'wallet': (bool),
'session': (bool),
'dht': (bool),
'hash_announcer': (bool),
'stream_identifier': (bool),
'file_manager': (bool),
'blob_manager': (bool),
'blockchain_headers': (bool),
'peer_protocol_server': (bool),
'reflector': (bool),
'upnp': (bool),
'exchange_rate_manager': (bool),
'startup_status': {
'code': status code,
'message': status message
},
'connection_status': {
'code': (str) connection status code,
'message': (str) connection status message
'code': connection status code,
'message': connection status message
},
'blockchain_headers': {
'downloading_headers': (bool),
'download_progress': (float) 0-100.0
'blockchain_status': {
'blocks': local blockchain height,
'blocks_behind': remote_height - local_height,
'best_blockhash': block hash of most recent block,
},
'wallet': {
'blocks': (int) local blockchain height,
'blocks_behind': (int) remote_height - local_height,
'best_blockhash': (str) block hash of most recent block,
'is_encrypted': (bool),
'is_locked': (bool),
},
'dht': {
'node_id': (str) lbry dht node id - hex encoded,
'peers_in_routing_table': (int) the number of peers in the routing table,
},
'blob_manager': {
'finished_blobs': (int) number of finished blobs in the blob manager,
},
'hash_announcer': {
'announce_queue_size': (int) number of blobs currently queued to be announced
},
'file_manager': {
'managed_files': (int) count of files in the file manager,
}
'wallet_is_encrypted': bool,
If given the session status option:
'session_status': {
'managed_blobs': count of blobs in the blob manager,
'managed_streams': count of streams in the file manager
'announce_queue_size': number of blobs currently queued to be announced
'should_announce_blobs': number of blobs that should be announced
}
If given the dht status option:
'dht_status': {
'kbps_received': current kbps receiving,
'kbps_sent': current kdps being sent,
'total_bytes_sent': total bytes sent,
'total_bytes_received': total bytes received,
'queries_received': number of queries received per second,
'queries_sent': number of queries sent per second,
'recent_contacts': count of recently contacted peers,
'unique_contacts': count of unique peers
},
}
</code></pre>
@@ -2184,7 +2089,7 @@ Options:
--uri=&lt;uri&gt; : (str) check availability for this uri
--search_timeout=&lt;search_timeout&gt; : (int) how long to search for peers for the blob
in the dht
--blob_timeout=&lt;blob_timeout&gt; : (int) how long to try downloading from a peer
--search_timeout=&lt;blob_timeout&gt; : (int) how long to try downloading from a peer
Returns:
(dict) {
@@ -2475,9 +2380,9 @@ Usage:
Options:
--amount=&lt;amount&gt; : (Decimal) amount of credit to send
--address=&lt;address&gt; : (str) address to send credits to
--claim_id=&lt;claim_id&gt; : (str) claim_id of the claim to send to tip to
--amount=&lt;amount&gt; : (float) amount of credit to send
--address=&lt;address&gt; : (str) address to send credits to
--claim_id=&lt;claim_id&gt; : (float) claim_id of the claim to send to tip to
Returns:
If sending to an address:
+89 -174
View File
@@ -295,13 +295,6 @@
blob_list
</a>
</li>
<li class="md-nav__item">
<a href="#blob_reflect" title="blob_reflect" class="md-nav__link">
blob_reflect
</a>
</li>
<li class="md-nav__item">
@@ -470,13 +463,6 @@
peer_list
</a>
</li>
<li class="md-nav__item">
<a href="#peer_ping" title="peer_ping" class="md-nav__link">
peer_ping
</a>
</li>
<li class="md-nav__item">
@@ -484,13 +470,6 @@
publish
</a>
</li>
<li class="md-nav__item">
<a href="#report_bug" title="report_bug" class="md-nav__link">
report_bug
</a>
</li>
<li class="md-nav__item">
@@ -730,13 +709,6 @@
blob_list
</a>
</li>
<li class="md-nav__item">
<a href="#blob_reflect" title="blob_reflect" class="md-nav__link">
blob_reflect
</a>
</li>
<li class="md-nav__item">
@@ -905,13 +877,6 @@
peer_list
</a>
</li>
<li class="md-nav__item">
<a href="#peer_ping" title="peer_ping" class="md-nav__link">
peer_ping
</a>
</li>
<li class="md-nav__item">
@@ -919,13 +884,6 @@
publish
</a>
</li>
<li class="md-nav__item">
<a href="#report_bug" title="report_bug" class="md-nav__link">
report_bug
</a>
</li>
<li class="md-nav__item">
@@ -1113,11 +1071,12 @@
<pre><code class="text">Announce blobs to the DHT
Args:
'blob_hash' : (str) announce a blob, specified by blob_hash
'stream_hash' : (str) announce all blobs associated with
stream_hash
'sd_hash' : (str) announce all blobs associated with
sd_hash and the sd_hash itself
'announce_all' : (bool) announce all the blobs possessed by user
'blob_hash' : (str) announce a blob, specified by blob_hash
'stream_hash' : (str) announce all blobs associated with
stream_hash
'sd_hash' : (str) announce all blobs associated with
sd_hash and the sd_hash itself
Returns:
(bool) true if successful
@@ -1185,16 +1144,6 @@ Returns:
(list) List of blob hashes
</code></pre>
<h2 id="blob_reflect">blob_reflect</h2>
<pre><code class="text">Reflects specified blobs
Args:
'reflector_server' : (str) reflector address
Returns:
(list) reflected blob hashes
</code></pre>
<h2 id="blob_reflect_all">blob_reflect_all</h2>
<pre><code class="text">Reflects all saved blobs
@@ -1230,7 +1179,7 @@ Returns:
<pre><code class="text">Import serialized channel signing information (to allow signing new claims to the channel)
Args:
'serialized_certificate_info' (required) : (str) certificate info
'serialized_certificate_info' : (str) certificate info
Returns:
(dict) Result dictionary
@@ -1380,7 +1329,7 @@ Returns:
'is_spent': (bool) true if claim is abandoned, false otherwise
'name': (str) name of the claim
'permanent_url': (str) permanent url of the claim,
'txid': (str) txid of the claim
'txid': (str) txid of the cliam
'nout': (int) nout of the claim
'value': (str) value of the claim
},
@@ -1479,12 +1428,12 @@ Returns:
<h2 id="cli_test_command">cli_test_command</h2>
<pre><code class="text">This command is only for testing the CLI argument parsing
Args:
'a_arg' : (bool) a arg
'b_arg' : (bool) b arg
'pos_arg' (required) : (int) pos arg
'pos_args' : (int) pos args
'pos_arg2' : (int) pos arg 2
'pos_arg3' : (int) pos arg 3
'a_arg' : a arg
'b_arg' : b arg
'pos_arg' (required) : pos arg
'pos_args' : pos args
'pos_arg2' : pos arg 2
'pos_arg3' : pos arg 3
Returns:
pos args
@@ -1552,9 +1501,6 @@ Args:
'claim_name' : (str) get file with matching claim name
'full_status' : (bool) full status, populate the
'message' and 'size' fields
'sort' : (str) sort by any property, like 'file_name'
or 'metadata.author'; to specify direction
append ',asc' or ',desc'
Returns:
(list) List of files
@@ -1678,19 +1624,7 @@ Args:
'timeout' : (int) peer search timeout in seconds
Returns:
(list) List of contact dictionaries {'host': &lt;peer ip&gt;, 'port': &lt;peer port&gt;, 'node_id': &lt;peer node id&gt;}
</code></pre>
<h2 id="peer_ping">peer_ping</h2>
<pre><code class="text">Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,
if not provided the peer is located first.
Args:
'address' : (str) ip address of the peer
'port' : (int) udp port of the peer
Returns:
(str) pong, or {'error': &lt;error message&gt;} if an error is encountered
(list) List of contacts
</code></pre>
<h2 id="publish">publish</h2>
@@ -1710,40 +1644,40 @@ fee, title, description, author, language, license, license_url, thumbnail, prev
or sources. Individual arguments will overwrite the fields specified in metadata argument.
Args:
'name' (required) : (str) name of the content
'bid' (required) : (decimal) amount to back the claim
'metadata' : (dict) ClaimDict to associate with the claim.
'file_path' : (str) path to file to be associated with name. If provided,
a lbry stream of this file will be used in 'sources'.
If no path is given but a sources dict is provided,
it will be used. If neither are provided, an
error is raised.
'fee' : (dict) Dictionary representing key fee to download content:
{
'currency': currency_symbol,
'amount': decimal,
'address': str, optional
}
supported currencies: LBC, USD, BTC
If an address is not provided a new one will be
automatically generated. Default fee is zero.
'title' : (str) title of the publication
'description' : (str) description of the publication
'author' : (str) author of the publication
'language' : (str) language of the publication
'license' : (str) publication license
'license_url' : (str) publication license url
'thumbnail' : (str) thumbnail url
'preview' : (str) preview url
'nsfw' : (bool) whether the content is nsfw
'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
'channel_name' : (str) name of the publisher channel name in the wallet
'channel_id' : (str) claim id of the publisher channel, does not check
for channel claim being in the wallet. This allows
publishing to a channel where only the certificate
private key is in the wallet.
'claim_address' : (str) address where the claim is sent to, if not specified
new address will automatically be created
'name' (required) : (str) name of the content
'bid' (required) : (float) amount to back the claim
'metadata' : (dict) ClaimDict to associate with the claim.
'file_path' : (str) path to file to be associated with name. If provided,
a lbry stream of this file will be used in 'sources'.
If no path is given but a sources dict is provided,
it will be used. If neither are provided, an
error is raised.
'fee' : (dict) Dictionary representing key fee to download content:
{
'currency': currency_symbol,
'amount': float,
'address': str, optional
}
supported currencies: LBC, USD, BTC
If an address is not provided a new one will be
automatically generated. Default fee is zero.
'title' : (str) title of the publication
'description' : (str) description of the publication
'author' : (str) author of the publication
'language' : (str) language of the publication
'license' : (str) publication license
'license_url' : (str) publication license url
'thumbnail' : (str) thumbnail url
'preview' : (str) preview url
'nsfw' : (bool) title of the publication
'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
'channel_name' : (str) name of the publisher channel name in the wallet
'channel_id' : (str) claim id of the publisher channel, does not check
for channel claim being in the wallet. This allows
publishing to a channel where only the certificate
private key is in the wallet.
'claim_address' : (str) address where the claim is sent to, if not specified
new address wil automatically be created
Returns:
(dict) Dictionary containing result of the claim
@@ -1751,21 +1685,11 @@ Returns:
'tx' : (str) hex encoded transaction
'txid' : (str) txid of resulting claim
'nout' : (int) nout of the resulting claim
'fee' : (decimal) fee paid for the claim transaction
'fee' : (float) fee paid for the claim transaction
'claim_id' : (str) claim ID of the resulting claim
}
</code></pre>
<h2 id="report_bug">report_bug</h2>
<pre><code class="text">Report a bug to slack
Args:
'message' (required) : (str) Description of the bug
Returns:
(bool) true if successful
</code></pre>
<h2 id="resolve">resolve</h2>
<pre><code class="text">Resolve given LBRY URIs
@@ -1855,7 +1779,6 @@ Returns:
&lt;bucket index&gt;: [
{
&quot;address&quot;: (str) peer address,
&quot;port&quot;: (int) peer udp port
&quot;node_id&quot;: (str) peer node id,
&quot;blobs&quot;: (list) blob hashes announced by peer
}
@@ -1915,58 +1838,50 @@ Returns:
<pre><code class="text">Get daemon status
Args:
None
'session_status' : (bool) include session status in results
'dht_status' : (bool) include dht network and peer status
Returns:
(dict) lbrynet-daemon status
{
'installation_id': (str) installation id - base58,
'is_running': (bool),
'lbry_id': lbry peer id, base58,
'installation_id': installation id, base58,
'is_running': bool,
'is_first_run': bool,
'skipped_components': (list) [names of skipped components (str)],
'startup_status': { Does not include components which have been skipped
'database': (bool),
'wallet': (bool),
'session': (bool),
'dht': (bool),
'hash_announcer': (bool),
'stream_identifier': (bool),
'file_manager': (bool),
'blob_manager': (bool),
'blockchain_headers': (bool),
'peer_protocol_server': (bool),
'reflector': (bool),
'upnp': (bool),
'exchange_rate_manager': (bool),
'startup_status': {
'code': status code,
'message': status message
},
'connection_status': {
'code': (str) connection status code,
'message': (str) connection status message
'code': connection status code,
'message': connection status message
},
'blockchain_headers': {
'downloading_headers': (bool),
'download_progress': (float) 0-100.0
'blockchain_status': {
'blocks': local blockchain height,
'blocks_behind': remote_height - local_height,
'best_blockhash': block hash of most recent block,
},
'wallet': {
'blocks': (int) local blockchain height,
'blocks_behind': (int) remote_height - local_height,
'best_blockhash': (str) block hash of most recent block,
'is_encrypted': (bool),
'is_locked': (bool),
},
'dht': {
'node_id': (str) lbry dht node id - hex encoded,
'peers_in_routing_table': (int) the number of peers in the routing table,
},
'blob_manager': {
'finished_blobs': (int) number of finished blobs in the blob manager,
},
'hash_announcer': {
'announce_queue_size': (int) number of blobs currently queued to be announced
},
'file_manager': {
'managed_files': (int) count of files in the file manager,
}
'wallet_is_encrypted': bool,
If given the session status option:
'session_status': {
'managed_blobs': count of blobs in the blob manager,
'managed_streams': count of streams in the file manager
'announce_queue_size': number of blobs currently queued to be announced
'should_announce_blobs': number of blobs that should be announced
}
If given the dht status option:
'dht_status': {
'kbps_received': current kbps receiving,
'kbps_sent': current kdps being sent,
'total_bytes_sent': total bytes sent,
'total_bytes_received': total bytes received,
'queries_received': number of queries received per second,
'queries_sent': number of queries sent per second,
'recent_contacts': count of recently contacted peers,
'unique_contacts': count of unique peers
},
}
</code></pre>
@@ -1977,7 +1892,7 @@ Args:
'uri' (required) : (str) check availability for this uri
'search_timeout' : (int) how long to search for peers for the blob
in the dht
'blob_timeout' : (int) how long to try downloading from a peer
'search_timeout' : (int) how long to try downloading from a peer
Returns:
(dict) {
@@ -2209,9 +2124,9 @@ to the owner of a claim specified by uri. A tip is a claim support where the rec
of the support is the claim address for the claim being supported.
Args:
'amount' (required) : (Decimal) amount of credit to send
'address' (required) : (str) address to send credits to
'claim_id' (required) : (str) claim_id of the claim to send to tip to
'amount' (required) : (float) amount of credit to send
'address' (required) : (str) address to send credits to
'claim_id' (required) : (float) claim_id of the claim to send to tip to
Returns:
If sending to an address:
+24 -54
View File
@@ -2,7 +2,7 @@
"docs": [
{
"location": "/",
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State Line truncated
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nArgs:\n 'announce_all' : (bool) announce all the blobs possessed by user\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nArgs:\n 'blob_hash' : (str) check availability for this blob hash\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nArgs:\n 'blob_hash' : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nArgs:\n 'blob_hash' (required) : (str) blob hash of the blob to get\n 'timeout' : (int) timeout in number of seconds\n 'encoding' : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n 'payment_rate_manager' : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nArgs:\n 'blockhash' (required) : (str) hash of the block to look up\n 'height' (required) : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nArgs:\n 'claim_id' (required) : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' : (str) certificate info\n\nReturns:\n (dict) Result dictionary\n\n\n\n\nchannel_list\n\n\nGet certificate claim infos for channels that can be published to\n\nArgs:\n None\n\nReturns:\n (list) ClaimDict, includes 'is_mine' field to indicate if the certificate claim\n is in the wallet.\n\n\n\n\nchannel_new\n\n\nGenerate a publisher key and create a new '@' prefixed certificate claim\n\nArgs:\n 'channel_name' (required) : (str) name of the channel prefixed with '@'\n 'amount' (required) : (float) bid amount on the channel\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }\n\n\n\n\nclaim_abandon\n\n\nAbandon a name and reclaim credits from the claim\n\nArgs:\n 'claim_id' : (str) claim_id of the claim to abandon\n 'txid' : (str) txid of the claim to abandon\n 'nout' : (int) nout of the claim to abandon\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n txid : (str) txid of resulting transaction\n fee : (float) fee paid for the transaction\n }\n\n\n\n\nclaim_list\n\n\nList current claims and information about them for a given name\n\nArgs:\n 'name' (required) : (str) name of the claim to list info about\n\nReturns:\n (dict) State of claims assigned for the name\n {\n 'claims': (list) list of claims for the Line truncated
"title": "API"
},
{
@@ -12,7 +12,7 @@
},
{
"location": "/#blob_announce",
"text": "Announce blobs to the DHT\n\nArgs:\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful",
"text": "Announce blobs to the DHT\n\nArgs:\n 'announce_all' : (bool) announce all the blobs possessed by user\n 'blob_hash' : (str) announce a blob, specified by blob_hash\n 'stream_hash' : (str) announce all blobs associated with\n stream_hash\n 'sd_hash' : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful",
"title": "blob_announce"
},
{
@@ -35,11 +35,6 @@
"text": "Returns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nArgs:\n 'needed' : (bool) only return needed blobs\n 'finished' : (bool) only return finished blobs\n 'uri' : (str) filter blobs by stream in a uri\n 'stream_hash' : (str) filter blobs by stream hash\n 'sd_hash' : (str) filter blobs by sd hash\n 'page_size' : (int) results page size\n 'page' : (int) page of results to return\n\nReturns:\n (list) List of blob hashes",
"title": "blob_list"
},
{
"location": "/#blob_reflect",
"text": "Reflects specified blobs\n\nArgs:\n 'reflector_server' : (str) reflector address\n\nReturns:\n (list) reflected blob hashes",
"title": "blob_reflect"
},
{
"location": "/#blob_reflect_all",
"text": "Reflects all saved blobs\n\nArgs:\n None\n\nReturns:\n (bool) true if successful",
@@ -57,7 +52,7 @@
},
{
"location": "/#channel_import",
"text": "Import serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' (required) : (str) certificate info\n\nReturns:\n (dict) Result dictionary",
"text": "Import serialized channel signing information (to allow signing new claims to the channel)\n\nArgs:\n 'serialized_certificate_info' : (str) certificate info\n\nReturns:\n (dict) Result dictionary",
"title": "channel_import"
},
{
@@ -87,7 +82,7 @@
},
{
"location": "/#claim_list_mine",
"text": "List my name claims\n\nArgs:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the claim\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
"text": "List my name claims\n\nArgs:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the cliam\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
"title": "claim_list_mine"
},
{
@@ -112,7 +107,7 @@
},
{
"location": "/#cli_test_command",
"text": "This command is only for testing the CLI argument parsing\nArgs:\n 'a_arg' : (bool) a arg\n 'b_arg' : (bool) b arg\n 'pos_arg' (required) : (int) pos arg\n 'pos_args' : (int) pos args\n 'pos_arg2' : (int) pos arg 2\n 'pos_arg3' : (int) pos arg 3\n\nReturns:\n pos args",
"text": "This command is only for testing the CLI argument parsing\nArgs:\n 'a_arg' : a arg\n 'b_arg' : b arg\n 'pos_arg' (required) : pos arg\n 'pos_args' : pos args\n 'pos_arg2' : pos arg 2\n 'pos_arg3' : pos arg 3\n\nReturns:\n pos args",
"title": "cli_test_command"
},
{
@@ -132,7 +127,7 @@
},
{
"location": "/#file_list",
"text": "List files limited by optional filters\n\nArgs:\n 'sd_hash' : (str) get file with matching sd hash\n 'file_name' : (str) get file with matching file name in the\n downloads folder\n 'stream_hash' : (str) get file with matching stream hash\n 'rowid' : (int) get file with matching row id\n 'claim_id' : (str) get file with matching claim id\n 'outpoint' : (str) get file with matching claim outpoint\n 'txid' : (str) get file with matching claim txid\n 'nout' : (int) get file with matching claim nout\n 'channel_claim_id' : (str) get file with matching channel claim id\n 'channel_name' : (str) get file with matching channel name\n 'claim_name' : (str) get file with matching claim name\n 'full_status' : (bool) full status, populate the\n 'message' and 'size' fields\n 'sort' : (str) sort by any property, like 'file_name'\n or 'metadata.author'; to specify direction\n append ',asc' or ',desc'\n\nReturns:\n (list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes, None if full_status is false,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) num_completed, None if full_status is false,\n 'blobs_in_stream': (int) None if full_status is false,\n 'status': (str) downloader status, None if full_status is false,\n 'claim_id': (str) None if full_status is false or if claim is not found,\n 'outpoint': (str) None if full_status is false or if claim is not found,\n 'txid': (str) None if full_status is false or if claim is not found,\n 'nout': (int) None if full_status is false or if claim is not found,\n 'metadata': (dict) None if full_status is false or if claim is not found,\n 'channel_claim_id': (str) None if full_status is false or if claim is not found or signed,\n 'channel_name': (str) None if full_status is false or if claim is not found or signed,\n 'claim_name': (str) None if full_status is false or if claim is not found\n },\n ]",
"text": "List files limited by optional filters\n\nArgs:\n 'sd_hash' : (str) get file with matching sd hash\n 'file_name' : (str) get file with matching file name in the\n downloads folder\n 'stream_hash' : (str) get file with matching stream hash\n 'rowid' : (int) get file with matching row id\n 'claim_id' : (str) get file with matching claim id\n 'outpoint' : (str) get file with matching claim outpoint\n 'txid' : (str) get file with matching claim txid\n 'nout' : (int) get file with matching claim nout\n 'channel_claim_id' : (str) get file with matching channel claim id\n 'channel_name' : (str) get file with matching channel name\n 'claim_name' : (str) get file with matching claim name\n 'full_status' : (bool) full status, populate the\n 'message' and 'size' fields\n\nReturns:\n (list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes, None if full_status is false,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) num_completed, None if full_status is false,\n 'blobs_in_stream': (int) None if full_status is false,\n 'status': (str) downloader status, None if full_status is false,\n 'claim_id': (str) None if full_status is false or if claim is not found,\n 'outpoint': (str) None if full_status is false or if claim is not found,\n 'txid': (str) None if full_status is false or if claim is not found,\n 'nout': (int) None if full_status is false or if claim is not found,\n 'metadata': (dict) None if full_status is false or if claim is not found,\n 'channel_claim_id': (str) None if full_status is false or if claim is not found or signed,\n 'channel_name': (str) None if full_status is false or if claim is not found or signed,\n 'claim_name': (str) None if full_status is false or if claim is not found\n },\n ]",
"title": "file_list"
},
{
@@ -157,24 +152,14 @@
},
{
"location": "/#peer_list",
"text": "Get peers for blob hash\n\nArgs:\n 'blob_hash' (required) : (str) find available peers for this blob hash\n 'timeout' : (int) peer search timeout in seconds\n\nReturns:\n (list) List of contact dictionaries {'host': peer ip , 'port': peer port , 'node_id': peer node id }",
"text": "Get peers for blob hash\n\nArgs:\n 'blob_hash' (required) : (str) find available peers for this blob hash\n 'timeout' : (int) peer search timeout in seconds\n\nReturns:\n (list) List of contacts",
"title": "peer_list"
},
{
"location": "/#peer_ping",
"text": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.\n\nArgs:\n 'address' : (str) ip address of the peer\n 'port' : (int) udp port of the peer\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
"title": "peer_ping"
},
{
"location": "/#publish",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (decimal) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) whether the content is nsfw\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nArgs:\n 'name' (required) : (str) name of the content\n 'bid' (required) : (float) amount to back the claim\n 'metadata' : (dict) ClaimDict to associate with the claim.\n 'file_path' : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n 'fee' : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': float,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n 'title' : (str) title of the publication\n 'description' : (str) description of the publication\n 'author' : (str) author of the publication\n 'language' : (str) language of the publication\n 'license' : (str) publication license\n 'license_url' : (str) publication license url\n 'thumbnail' : (str) thumbnail url\n 'preview' : (str) preview url\n 'nsfw' : (bool) title of the publication\n 'sources' : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n 'channel_name' : (str) name of the publisher channel name in the wallet\n 'channel_id' : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n 'claim_address' : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"title": "publish"
},
{
"location": "/#report_bug",
"text": "Report a bug to slack\n\nArgs:\n 'message' (required) : (str) Description of the bug\n\nReturns:\n (bool) true if successful",
"title": "report_bug"
},
{
"location": "/#resolve",
"text": "Resolve given LBRY URIs\n\nArgs:\n 'force' : (bool) force refresh and ignore cache\n 'uri' (required) : (str) uri to resolve\n 'uris' : (list) uris to resolve\n\nReturns:\n Dictionary of results, keyed by uri\n ' uri ': {\n If a resolution error occurs:\n 'error': Error message\n\n If the uri resolves to a channel or a claim in a channel:\n 'certificate': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the certificate claim,\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}],\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n\n If the uri resolves to a channel:\n 'claims_in_channel': (int) number of claims in the channel,\n\n If the uri resolves to a claim:\n 'claim': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the claim,\n 'channel_name': (str) channel name if claim is in a channel\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}]\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n }",
@@ -187,7 +172,7 @@
},
{
"location": "/#routing_table_get",
"text": "Get DHT routing information\n\nArgs:\n None\n\nReturns:\n (dict) dictionary containing routing and contact information\n {\n buckets : {\n bucket index : [\n {\n address : (str) peer address,\n port : (int) peer udp port\n node_id : (str) peer node id,\n blobs : (list) blob hashes announced by peer\n }\n ]\n },\n contacts : (list) contact node ids,\n blob_hashes : (list) all of the blob hashes stored by peers in the list of buckets,\n node_id : (str) the local dht node id\n }",
"text": "Get DHT routing information\n\nArgs:\n None\n\nReturns:\n (dict) dictionary containing routing and contact information\n {\n buckets : {\n bucket index : [\n {\n address : (str) peer address,\n node_id : (str) peer node id,\n blobs : (list) blob hashes announced by peer\n }\n ]\n },\n contacts : (list) contact node ids,\n blob_hashes : (list) all of the blob hashes stored by peers in the list of buckets,\n node_id : (str) the local dht node id\n }",
"title": "routing_table_get"
},
{
@@ -202,12 +187,12 @@
},
{
"location": "/#status",
"text": "Get daemon status\n\nArgs:\n None\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }",
"text": "Get daemon status\n\nArgs:\n 'session_status' : (bool) include session status in results\n 'dht_status' : (bool) include dht network and peer status\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'lbry_id': lbry peer id, base58,\n 'installation_id': installation id, base58,\n 'is_running': bool,\n 'is_first_run': bool,\n 'startup_status': {\n 'code': status code,\n 'message': status message\n },\n 'connection_status': {\n 'code': connection status code,\n 'message': connection status message\n },\n 'blockchain_status': {\n 'blocks': local blockchain height,\n 'blocks_behind': remote_height - local_height,\n 'best_blockhash': block hash of most recent block,\n },\n 'wallet_is_encrypted': bool,\n\n If given the session status option:\n 'session_status': {\n 'managed_blobs': count of blobs in the blob manager,\n 'managed_streams': count of streams in the file manager\n 'announce_queue_size': number of blobs currently queued to be announced\n 'should_announce_blobs': number of blobs that should be announced\n }\n\n If given the dht status option:\n 'dht_status': {\n 'kbps_received': current kbps receiving,\n 'kbps_sent': current kdps being sent,\n 'total_bytes_sent': total bytes sent,\n 'total_bytes_received': total bytes received,\n 'queries_received': number of queries received per second,\n 'queries_sent': number of queries sent per second,\n 'recent_contacts': count of recently contacted peers,\n 'unique_contacts': count of unique peers\n },\n }",
"title": "status"
},
{
"location": "/#stream_availability",
"text": "Get stream availability for lbry uri\n\nArgs:\n 'uri' (required) : (str) check availability for this uri\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'blob_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n 'is_available': bool ,\n 'did_decode': bool ,\n 'did_resolve': bool ,\n 'is_stream': bool ,\n 'num_blobs_in_stream': int ,\n 'sd_hash': str ,\n 'sd_blob_availability': dict see `blob_availability`,\n 'head_blob_hash': str ,\n 'head_blob_availability': dict see `blob_availability`,\n 'use_upnp': bool ,\n 'upnp_redirect_is_set': bool ,\n 'error': None | str error message\n }",
"text": "Get stream availability for lbry uri\n\nArgs:\n 'uri' (required) : (str) check availability for this uri\n 'search_timeout' : (int) how long to search for peers for the blob\n in the dht\n 'search_timeout' : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n 'is_available': bool ,\n 'did_decode': bool ,\n 'did_resolve': bool ,\n 'is_stream': bool ,\n 'num_blobs_in_stream': int ,\n 'sd_hash': str ,\n 'sd_blob_availability': dict see `blob_availability`,\n 'head_blob_hash': str ,\n 'head_blob_availability': dict see `blob_availability`,\n 'use_upnp': bool ,\n 'upnp_redirect_is_set': bool ,\n 'error': None | str error message\n }",
"title": "stream_availability"
},
{
@@ -277,7 +262,7 @@
},
{
"location": "/#wallet_send",
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nArgs:\n 'amount' (required) : (Decimal) amount of credit to send\n 'address' (required) : (str) address to send credits to\n 'claim_id' (required) : (str) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nArgs:\n 'amount' (required) : (float) amount of credit to send\n 'address' (required) : (str) address to send credits to\n 'claim_id' (required) : (float) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
"title": "wallet_send"
},
{
@@ -292,7 +277,7 @@
},
{
"location": "/cli/",
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n] | [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect\n\n\nReflects specified blobs\n\nUsage:\n blob_reflect (\nblob_hashes\n...) [--reflector_server=\nreflector_server\n]\n\n\nOptions:\n --reflector_server=\nreflector_server\n : (str) reflector address\n\nReturns:\n (list) reflected blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export informatiLine truncated
"text": "LBRY JSON-RPC API Documentation\n\n\nblob_announce\n\n\nAnnounce blobs to the DHT\n\nUsage:\n blob_announce [--announce_all] [\nblob_hash\n | --blob_hash=\nblob_hash\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n\n\nOptions:\n --announce_all=\nannounce_all\n : (bool) announce all the blobs possessed by user\n --blob_hash=\nblob_hash\n : (str) announce a blob, specified by blob_hash\n --stream_hash=\nstream_hash\n : (str) announce all blobs associated with\n stream_hash\n --sd_hash=\nsd_hash\n : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful\n\n\n\n\nblob_availability\n\n\nGet blob availability\n\nUsage:\n blob_availability (\nblob_hash\n) [\nsearch_timeout\n | --search_timeout=\nsearch_timeout\n]\n [\nblob_timeout\n | --blob_timeout=\nblob_timeout\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) check availability for this blob hash\n --search_timeout=\nsearch_timeout\n : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout=\nblob_timeout\n : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n \nis_available\n: \nbool, true if blob is available from a peer from peer list\n\n \nreachable_peers\n: [\nip\n:\nport\n],\n \nunreachable_peers\n: [\nip\n:\nport\n]\n }\n\n\n\n\nblob_delete\n\n\nDelete a blob\n\nUsage:\n blob_delete (\nblob_hash\n | --blob_hash=\nblob_hash)\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to delete\n\nReturns:\n (str) Success/fail message\n\n\n\n\nblob_get\n\n\nDownload and return a blob\n\nUsage:\n blob_get (\nblob_hash\n | --blob_hash=\nblob_hash\n) [--timeout=\ntimeout\n]\n [--encoding=\nencoding\n] [--payment_rate_manager=\npayment_rate_manager\n]\n\n\nOptions:\n --blob_hash=\nblob_hash\n : (str) blob hash of the blob to get\n --timeout=\ntimeout\n : (int) timeout in number of seconds\n --encoding=\nencoding\n : (str) by default no attempt at decoding\n is made, can be set to one of the\n following decoders:\n 'json'\n --payment_rate_manager=\npayment_rate_manager\n : (str) if not given the default payment rate\n manager will be used.\n supported alternative rate managers:\n 'only-free'\n\nReturns:\n (str) Success/Fail message or (dict) decoded data\n\n\n\n\nblob_list\n\n\nReturns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [\nuri\n | --uri=\nuri\n]\n [\nstream_hash\n | --stream_hash=\nstream_hash\n]\n [\nsd_hash\n | --sd_hash=\nsd_hash\n]\n [\npage_size\n | --page_size=\npage_size\n]\n [\npage\n | --page=\npage\n]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri=\nuri\n : (str) filter blobs by stream in a uri\n --stream_hash=\nstream_hash\n : (str) filter blobs by stream hash\n --sd_hash=\nsd_hash\n : (str) filter blobs by sd hash\n --page_size=\npage_size\n : (int) results page size\n --page=\npage\n : (int) page of results to return\n\nReturns:\n (list) List of blob hashes\n\n\n\n\nblob_reflect_all\n\n\nReflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful\n\n\n\n\nblock_show\n\n\nGet contents of a block\n\nUsage:\n block_show (\nblockhash\n | --blockhash=\nblockhash\n) | (\nheight\n | --height=\nheight\n)\n\n\nOptions:\n --blockhash=\nblockhash\n : (str) hash of the block to look up\n --height=\nheight\n : (int) height of the block to look up\n\nReturns:\n (dict) Requested block\n\n\n\n\nchannel_export\n\n\nExport serialized channel signing information for a given certificate claim id\n\nUsage:\n channel_export (\nclaim_id\n | --claim_id=\nclaim_id\n)\n\n\nOptions:\n --claim_id=\nclaim_id\n : (str) Claim ID to export information about\n\nReturns:\n (str) Serialized certificate information\n\n\n\n\nchannel_import\n\n\nImport serialized channel signing informatLine truncated
"title": "CLI"
},
{
@@ -302,7 +287,7 @@
},
{
"location": "/cli/#blob_announce",
"text": "Announce blobs to the DHT\n\nUsage:\n blob_announce [ blob_hash | --blob_hash= blob_hash ]\n [ stream_hash | --stream_hash= stream_hash ] | [ sd_hash | --sd_hash= sd_hash ]\n\n\nOptions:\n --blob_hash= blob_hash : (str) announce a blob, specified by blob_hash\n --stream_hash= stream_hash : (str) announce all blobs associated with\n stream_hash\n --sd_hash= sd_hash : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful",
"text": "Announce blobs to the DHT\n\nUsage:\n blob_announce [--announce_all] [ blob_hash | --blob_hash= blob_hash ]\n [ stream_hash | --stream_hash= stream_hash ]\n [ sd_hash | --sd_hash= sd_hash ]\n\n\nOptions:\n --announce_all= announce_all : (bool) announce all the blobs possessed by user\n --blob_hash= blob_hash : (str) announce a blob, specified by blob_hash\n --stream_hash= stream_hash : (str) announce all blobs associated with\n stream_hash\n --sd_hash= sd_hash : (str) announce all blobs associated with\n sd_hash and the sd_hash itself\n\nReturns:\n (bool) true if successful",
"title": "blob_announce"
},
{
@@ -325,11 +310,6 @@
"text": "Returns blob hashes. If not given filters, returns all blobs known by the blob manager\n\nUsage:\n blob_list [--needed] [--finished] [ uri | --uri= uri ]\n [ stream_hash | --stream_hash= stream_hash ]\n [ sd_hash | --sd_hash= sd_hash ]\n [ page_size | --page_size= page_size ]\n [ page | --page= page ]\n\n\nOptions:\n --needed : (bool) only return needed blobs\n --finished : (bool) only return finished blobs\n --uri= uri : (str) filter blobs by stream in a uri\n --stream_hash= stream_hash : (str) filter blobs by stream hash\n --sd_hash= sd_hash : (str) filter blobs by sd hash\n --page_size= page_size : (int) results page size\n --page= page : (int) page of results to return\n\nReturns:\n (list) List of blob hashes",
"title": "blob_list"
},
{
"location": "/cli/#blob_reflect",
"text": "Reflects specified blobs\n\nUsage:\n blob_reflect ( blob_hashes ...) [--reflector_server= reflector_server ]\n\n\nOptions:\n --reflector_server= reflector_server : (str) reflector address\n\nReturns:\n (list) reflected blob hashes",
"title": "blob_reflect"
},
{
"location": "/cli/#blob_reflect_all",
"text": "Reflects all saved blobs\n\nUsage:\n blob_reflect_all\n\n\nOptions:\n None\n\nReturns:\n (bool) true if successful",
@@ -347,7 +327,7 @@
},
{
"location": "/cli/#channel_import",
"text": "Import serialized channel signing information (to allow signing new claims to the channel)\n\nUsage:\n channel_import ( serialized_certificate_info | --serialized_certificate_info= serialized_certificate_info )\n\n\nOptions:\n --serialized_certificate_info= serialized_certificate_info : (str) certificate info\n\nReturns:\n (dict) Result dictionary",
"text": "Import serialized channel signing information (to allow signing new claims to the channel)\n\nUsage:\n channel_import ( serialized_certificate_info |\n --serialized_certificate_info= serialized_certificate_info )\n\n\nOptions:\n --serialized_certificate_info= serialized_certificate_info : (str) certificate info\n\nReturns:\n (dict) Result dictionary",
"title": "channel_import"
},
{
@@ -377,7 +357,7 @@
},
{
"location": "/cli/#claim_list_mine",
"text": "List my name claims\n\nUsage:\n claim_list_mine\n\n\nOptions:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the claim\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
"text": "List my name claims\n\nUsage:\n claim_list_mine\n\n\nOptions:\n None\n\nReturns:\n (list) List of name claims owned by user\n [\n {\n 'address': (str) address that owns the claim\n 'amount': (float) amount assigned to the claim\n 'blocks_to_expiration': (int) number of blocks until it expires\n 'category': (str) claim , update , or support \n 'claim_id': (str) claim ID of the claim\n 'confirmations': (int) number of blocks of confirmations for the claim\n 'expiration_height': (int) the block height which the claim will expire\n 'expired': (bool) true if expired, false otherwise\n 'height': (int) height of the block containing the claim\n 'is_spent': (bool) true if claim is abandoned, false otherwise\n 'name': (str) name of the claim\n 'permanent_url': (str) permanent url of the claim,\n 'txid': (str) txid of the cliam\n 'nout': (int) nout of the claim\n 'value': (str) value of the claim\n },\n ]",
"title": "claim_list_mine"
},
{
@@ -402,7 +382,7 @@
},
{
"location": "/cli/#cli_test_command",
"text": "This command is only for testing the CLI argument parsing\nUsage:\n cli_test_command [--a_arg] [--b_arg] ( pos_arg | --pos_arg= pos_arg )\n [ pos_args ...] [--pos_arg2= pos_arg2 ]\n [--pos_arg3= pos_arg3 ]\n\n\nOptions:\n --a_arg : (bool) a arg\n --b_arg : (bool) b arg\n --pos_arg= pos_arg : (int) pos arg\n --pos_args= pos_args : (int) pos args\n --pos_arg2= pos_arg2 : (int) pos arg 2\n --pos_arg3= pos_arg3 : (int) pos arg 3\n\nReturns:\n pos args",
"text": "This command is only for testing the CLI argument parsing\nUsage:\n cli_test_command [--a_arg] [--b_arg] ( pos_arg | --pos_arg= pos_arg )\n [ pos_args ...] [--pos_arg2= pos_arg2 ]\n [--pos_arg3= pos_arg3 ]\n\n\nOptions:\n --a_arg : a arg\n --b_arg : b arg\n --pos_arg= pos_arg : pos arg\n --pos_args= pos_args : pos args\n --pos_arg2= pos_arg2 : pos arg 2\n --pos_arg3= pos_arg3 : pos arg 3\n\nReturns:\n pos args",
"title": "cli_test_command"
},
{
@@ -422,7 +402,7 @@
},
{
"location": "/cli/#file_list",
"text": "List files limited by optional filters\n\nUsage:\n file_list [--sd_hash= sd_hash ] [--file_name= file_name ] [--stream_hash= stream_hash ]\n [--rowid= rowid ] [--claim_id= claim_id ] [--outpoint= outpoint ] [--txid= txid ] [--nout= nout ]\n [--channel_claim_id= channel_claim_id ] [--channel_name= channel_name ]\n [--claim_name= claim_name ] [--full_status] [--sort= sort_method ...]\n\n\nOptions:\n --sd_hash= sd_hash : (str) get file with matching sd hash\n --file_name= file_name : (str) get file with matching file name in the\n downloads folder\n --stream_hash= stream_hash : (str) get file with matching stream hash\n --rowid= rowid : (int) get file with matching row id\n --claim_id= claim_id : (str) get file with matching claim id\n --outpoint= outpoint : (str) get file with matching claim outpoint\n --txid= txid : (str) get file with matching claim txid\n --nout= nout : (int) get file with matching claim nout\n --channel_claim_id= channel_claim_id : (str) get file with matching channel claim id\n --channel_name= channel_name : (str) get file with matching channel name\n --claim_name= claim_name : (str) get file with matching claim name\n --full_status : (bool) full status, populate the\n 'message' and 'size' fields\n --sort= sort_method : (str) sort by any property, like 'file_name'\n or 'metadata.author'; to specify direction\n append ',asc' or ',desc'\n\nReturns:\n (list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes, None if full_status is false,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) num_completed, None if full_status is false,\n 'blobs_in_stream': (int) None if full_status is false,\n 'status': (str) downloader status, None if full_status is false,\n 'claim_id': (str) None if full_status is false or if claim is not found,\n 'outpoint': (str) None if full_status is false or if claim is not found,\n 'txid': (str) None if full_status is false or if claim is not found,\n 'nout': (int) None if full_status is false or if claim is not found,\n 'metadata': (dict) None if full_status is false or if claim is not found,\n 'channel_claim_id': (str) None if full_status is false or if claim is not found or signed,\n 'channel_name': (str) None if full_status is false or if claim is not found or signed,\n 'claim_name': (str) None if full_status is false or if claim is not found\n },\n ]",
"text": "List files limited by optional filters\n\nUsage:\n file_list [--sd_hash= sd_hash ] [--file_name= file_name ] [--stream_hash= stream_hash ]\n [--rowid= rowid ] [--claim_id= claim_id ] [--outpoint= outpoint ] [--txid= txid ] [--nout= nout ]\n [--channel_claim_id= channel_claim_id ] [--channel_name= channel_name ]\n [--claim_name= claim_name ] [--full_status]\n\n\nOptions:\n --sd_hash= sd_hash : (str) get file with matching sd hash\n --file_name= file_name : (str) get file with matching file name in the\n downloads folder\n --stream_hash= stream_hash : (str) get file with matching stream hash\n --rowid= rowid : (int) get file with matching row id\n --claim_id= claim_id : (str) get file with matching claim id\n --outpoint= outpoint : (str) get file with matching claim outpoint\n --txid= txid : (str) get file with matching claim txid\n --nout= nout : (int) get file with matching claim nout\n --channel_claim_id= channel_claim_id : (str) get file with matching channel claim id\n --channel_name= channel_name : (str) get file with matching channel name\n --claim_name= claim_name : (str) get file with matching claim name\n --full_status : (bool) full status, populate the\n 'message' and 'size' fields\n\nReturns:\n (list) List of files\n\n [\n {\n 'completed': (bool) true if download is completed,\n 'file_name': (str) name of file,\n 'download_directory': (str) download directory,\n 'points_paid': (float) credit paid to download file,\n 'stopped': (bool) true if download is stopped,\n 'stream_hash': (str) stream hash of file,\n 'stream_name': (str) stream name ,\n 'suggested_file_name': (str) suggested file name,\n 'sd_hash': (str) sd hash of file,\n 'download_path': (str) download path of file,\n 'mime_type': (str) mime type of file,\n 'key': (str) key attached to file,\n 'total_bytes': (int) file size in bytes, None if full_status is false,\n 'written_bytes': (int) written size in bytes,\n 'blobs_completed': (int) num_completed, None if full_status is false,\n 'blobs_in_stream': (int) None if full_status is false,\n 'status': (str) downloader status, None if full_status is false,\n 'claim_id': (str) None if full_status is false or if claim is not found,\n 'outpoint': (str) None if full_status is false or if claim is not found,\n 'txid': (str) None if full_status is false or if claim is not found,\n 'nout': (int) None if full_status is false or if claim is not found,\n 'metadata': (dict) None if full_status is false or if claim is not found,\n 'channel_claim_id': (str) None if full_status is false or if claim is not found or signed,\n 'channel_name': (str) None if full_status is false or if claim is not found or signed,\n 'claim_name': (str) None if full_status is false or if claim is not found\n },\n ]",
"title": "file_list"
},
{
@@ -447,24 +427,14 @@
},
{
"location": "/cli/#peer_list",
"text": "Get peers for blob hash\n\nUsage:\n peer_list ( blob_hash | --blob_hash= blob_hash ) [ timeout | --timeout= timeout ]\n\n\nOptions:\n --blob_hash= blob_hash : (str) find available peers for this blob hash\n --timeout= timeout : (int) peer search timeout in seconds\n\nReturns:\n (list) List of contact dictionaries {'host': peer ip , 'port': peer port , 'node_id': peer node id }",
"text": "Get peers for blob hash\n\nUsage:\n peer_list ( blob_hash | --blob_hash= blob_hash ) [ timeout | --timeout= timeout ]\n\n\nOptions:\n --blob_hash= blob_hash : (str) find available peers for this blob hash\n --timeout= timeout : (int) peer search timeout in seconds\n\nReturns:\n (list) List of contacts",
"title": "peer_list"
},
{
"location": "/cli/#peer_ping",
"text": "Send a kademlia ping to the specified peer. If address and port are provided the peer is directly pinged,\nif not provided the peer is located first.\n\nUsage:\n peer_ping ( node_id | --node_id= node_id ) [ address | --address= address ] [ port | --port= port ]\n\n\nOptions:\n --address= address : (str) ip address of the peer\n --port= port : (int) udp port of the peer\n\nReturns:\n (str) pong, or {'error': error message } if an error is encountered",
"title": "peer_ping"
},
{
"location": "/cli/#publish",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (decimal) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': decimal,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) whether the content is nsfw\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address will automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (decimal) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"text": "Make a new name claim and publish associated data to lbrynet,\nupdate over existing claim if user already has a claim for name.\n\nFields required in the final Metadata are:\n 'title'\n 'description'\n 'author'\n 'language'\n 'license'\n 'nsfw'\n\nMetadata can be set by either using the metadata argument or by setting individual arguments\nfee, title, description, author, language, license, license_url, thumbnail, preview, nsfw,\nor sources. Individual arguments will overwrite the fields specified in metadata argument.\n\nUsage:\n publish ( name | --name= name ) ( bid | --bid= bid ) [--metadata= metadata ]\n [--file_path= file_path ] [--fee= fee ] [--title= title ]\n [--description= description ] [--author= author ] [--language= language ]\n [--license= license ] [--license_url= license_url ] [--thumbnail= thumbnail ]\n [--preview= preview ] [--nsfw= nsfw ] [--sources= sources ]\n [--channel_name= channel_name ] [--channel_id= channel_id ]\n [--claim_address= claim_address ] [--change_address= change_address ]\n\n\nOptions:\n --name= name : (str) name of the content\n --bid= bid : (float) amount to back the claim\n --metadata= metadata : (dict) ClaimDict to associate with the claim.\n --file_path= file_path : (str) path to file to be associated with name. If provided,\n a lbry stream of this file will be used in 'sources'.\n If no path is given but a sources dict is provided,\n it will be used. If neither are provided, an\n error is raised.\n --fee= fee : (dict) Dictionary representing key fee to download content:\n {\n 'currency': currency_symbol,\n 'amount': float,\n 'address': str, optional\n }\n supported currencies: LBC, USD, BTC\n If an address is not provided a new one will be\n automatically generated. Default fee is zero.\n --title= title : (str) title of the publication\n --description= description : (str) description of the publication\n --author= author : (str) author of the publication\n --language= language : (str) language of the publication\n --license= license : (str) publication license\n --license_url= license_url : (str) publication license url\n --thumbnail= thumbnail : (str) thumbnail url\n --preview= preview : (str) preview url\n --nsfw= nsfw : (bool) title of the publication\n --sources= sources : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file\n --channel_name= channel_name : (str) name of the publisher channel name in the wallet\n --channel_id= channel_id : (str) claim id of the publisher channel, does not check\n for channel claim being in the wallet. This allows\n publishing to a channel where only the certificate\n private key is in the wallet.\n --claim_address= claim_address : (str) address where the claim is sent to, if not specified\n new address wil automatically be created\n\nReturns:\n (dict) Dictionary containing result of the claim\n {\n 'tx' : (str) hex encoded transaction\n 'txid' : (str) txid of resulting claim\n 'nout' : (int) nout of the resulting claim\n 'fee' : (float) fee paid for the claim transaction\n 'claim_id' : (str) claim ID of the resulting claim\n }",
"title": "publish"
},
{
"location": "/cli/#report_bug",
"text": "Report a bug to slack\n\nUsage:\n report_bug ( message | --message= message )\n\n\nOptions:\n --message= message : (str) Description of the bug\n\nReturns:\n (bool) true if successful",
"title": "report_bug"
},
{
"location": "/cli/#resolve",
"text": "Resolve given LBRY URIs\n\nUsage:\n resolve [--force] ( uri | --uri= uri ) [ uris ...]\n\n\nOptions:\n --force : (bool) force refresh and ignore cache\n --uri= uri : (str) uri to resolve\n --uris= uris : (list) uris to resolve\n\nReturns:\n Dictionary of results, keyed by uri\n ' uri ': {\n If a resolution error occurs:\n 'error': Error message\n\n If the uri resolves to a channel or a claim in a channel:\n 'certificate': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the certificate claim,\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}],\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n\n If the uri resolves to a channel:\n 'claims_in_channel': (int) number of claims in the channel,\n\n If the uri resolves to a claim:\n 'claim': {\n 'address': (str) claim address,\n 'amount': (float) claim amount,\n 'effective_amount': (float) claim amount including supports,\n 'claim_id': (str) claim id,\n 'claim_sequence': (int) claim sequence number,\n 'decoded_claim': (bool) whether or not the claim value was decoded,\n 'height': (int) claim height,\n 'depth': (int) claim depth,\n 'has_signature': (bool) included if decoded_claim\n 'name': (str) claim name,\n 'permanent_url': (str) permanent url of the claim,\n 'channel_name': (str) channel name if claim is in a channel\n 'supports: (list) list of supports [{'txid': (str) txid,\n 'nout': (int) nout,\n 'amount': (float) amount}]\n 'txid': (str) claim txid,\n 'nout': (str) claim nout,\n 'signature_is_valid': (bool), included if has_signature,\n 'value': ClaimDict if decoded, otherwise hex string\n }\n }",
@@ -477,7 +447,7 @@
},
{
"location": "/cli/#routing_table_get",
"text": "Get DHT routing information\n\nUsage:\n routing_table_get\n\n\nOptions:\n None\n\nReturns:\n (dict) dictionary containing routing and contact information\n {\n buckets : {\n bucket index : [\n {\n address : (str) peer address,\n port : (int) peer udp port\n node_id : (str) peer node id,\n blobs : (list) blob hashes announced by peer\n }\n ]\n },\n contacts : (list) contact node ids,\n blob_hashes : (list) all of the blob hashes stored by peers in the list of buckets,\n node_id : (str) the local dht node id\n }",
"text": "Get DHT routing information\n\nUsage:\n routing_table_get\n\n\nOptions:\n None\n\nReturns:\n (dict) dictionary containing routing and contact information\n {\n buckets : {\n bucket index : [\n {\n address : (str) peer address,\n node_id : (str) peer node id,\n blobs : (list) blob hashes announced by peer\n }\n ]\n },\n contacts : (list) contact node ids,\n blob_hashes : (list) all of the blob hashes stored by peers in the list of buckets,\n node_id : (str) the local dht node id\n }",
"title": "routing_table_get"
},
{
@@ -492,12 +462,12 @@
},
{
"location": "/cli/#status",
"text": "Get daemon status\n\nUsage:\n status\n\n\nOptions:\n None\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'installation_id': (str) installation id - base58,\n 'is_running': (bool),\n 'is_first_run': bool,\n 'skipped_components': (list) [names of skipped components (str)],\n 'startup_status': { Does not include components which have been skipped\n 'database': (bool),\n 'wallet': (bool),\n 'session': (bool),\n 'dht': (bool),\n 'hash_announcer': (bool),\n 'stream_identifier': (bool),\n 'file_manager': (bool),\n 'blob_manager': (bool),\n 'blockchain_headers': (bool),\n 'peer_protocol_server': (bool),\n 'reflector': (bool),\n 'upnp': (bool),\n 'exchange_rate_manager': (bool),\n },\n 'connection_status': {\n 'code': (str) connection status code,\n 'message': (str) connection status message\n },\n 'blockchain_headers': {\n 'downloading_headers': (bool),\n 'download_progress': (float) 0-100.0\n },\n 'wallet': {\n 'blocks': (int) local blockchain height,\n 'blocks_behind': (int) remote_height - local_height,\n 'best_blockhash': (str) block hash of most recent block,\n 'is_encrypted': (bool),\n 'is_locked': (bool),\n },\n 'dht': {\n 'node_id': (str) lbry dht node id - hex encoded,\n 'peers_in_routing_table': (int) the number of peers in the routing table,\n },\n 'blob_manager': {\n 'finished_blobs': (int) number of finished blobs in the blob manager,\n },\n 'hash_announcer': {\n 'announce_queue_size': (int) number of blobs currently queued to be announced\n },\n 'file_manager': {\n 'managed_files': (int) count of files in the file manager,\n }\n }",
"text": "Get daemon status\n\nUsage:\n status [--session_status] [--dht_status]\n\n\nOptions:\n --session_status : (bool) include session status in results\n --dht_status : (bool) include dht network and peer status\n\nReturns:\n (dict) lbrynet-daemon status\n {\n 'lbry_id': lbry peer id, base58,\n 'installation_id': installation id, base58,\n 'is_running': bool,\n 'is_first_run': bool,\n 'startup_status': {\n 'code': status code,\n 'message': status message\n },\n 'connection_status': {\n 'code': connection status code,\n 'message': connection status message\n },\n 'blockchain_status': {\n 'blocks': local blockchain height,\n 'blocks_behind': remote_height - local_height,\n 'best_blockhash': block hash of most recent block,\n },\n 'wallet_is_encrypted': bool,\n\n If given the session status option:\n 'session_status': {\n 'managed_blobs': count of blobs in the blob manager,\n 'managed_streams': count of streams in the file manager\n 'announce_queue_size': number of blobs currently queued to be announced\n 'should_announce_blobs': number of blobs that should be announced\n }\n\n If given the dht status option:\n 'dht_status': {\n 'kbps_received': current kbps receiving,\n 'kbps_sent': current kdps being sent,\n 'total_bytes_sent': total bytes sent,\n 'total_bytes_received': total bytes received,\n 'queries_received': number of queries received per second,\n 'queries_sent': number of queries sent per second,\n 'recent_contacts': count of recently contacted peers,\n 'unique_contacts': count of unique peers\n },\n }",
"title": "status"
},
{
"location": "/cli/#stream_availability",
"text": "Get stream availability for lbry uri\n\nUsage:\n stream_availability ( uri | --uri= uri )\n [ search_timeout | --search_timeout= search_timeout ]\n [ blob_timeout | --blob_timeout= blob_timeout ]\n\n\nOptions:\n --uri= uri : (str) check availability for this uri\n --search_timeout= search_timeout : (int) how long to search for peers for the blob\n in the dht\n --blob_timeout= blob_timeout : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n 'is_available': bool ,\n 'did_decode': bool ,\n 'did_resolve': bool ,\n 'is_stream': bool ,\n 'num_blobs_in_stream': int ,\n 'sd_hash': str ,\n 'sd_blob_availability': dict see `blob_availability`,\n 'head_blob_hash': str ,\n 'head_blob_availability': dict see `blob_availability`,\n 'use_upnp': bool ,\n 'upnp_redirect_is_set': bool ,\n 'error': None | str error message\n }",
"text": "Get stream availability for lbry uri\n\nUsage:\n stream_availability ( uri | --uri= uri )\n [ search_timeout | --search_timeout= search_timeout ]\n [ blob_timeout | --blob_timeout= blob_timeout ]\n\n\nOptions:\n --uri= uri : (str) check availability for this uri\n --search_timeout= search_timeout : (int) how long to search for peers for the blob\n in the dht\n --search_timeout= blob_timeout : (int) how long to try downloading from a peer\n\nReturns:\n (dict) {\n 'is_available': bool ,\n 'did_decode': bool ,\n 'did_resolve': bool ,\n 'is_stream': bool ,\n 'num_blobs_in_stream': int ,\n 'sd_hash': str ,\n 'sd_blob_availability': dict see `blob_availability`,\n 'head_blob_hash': str ,\n 'head_blob_availability': dict see `blob_availability`,\n 'use_upnp': bool ,\n 'upnp_redirect_is_set': bool ,\n 'error': None | str error message\n }",
"title": "stream_availability"
},
{
@@ -567,7 +537,7 @@
},
{
"location": "/cli/#wallet_send",
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nUsage:\n wallet_send ( amount | --amount= amount )\n (( address | --address= address ) | ( claim_id | --claim_id= claim_id ))\n\n\nOptions:\n --amount= amount : (Decimal) amount of credit to send\n --address= address : (str) address to send credits to\n --claim_id= claim_id : (str) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
"text": "Send credits. If given an address, send credits to it. If given a claim id, send a tip\nto the owner of a claim specified by uri. A tip is a claim support where the recipient\nof the support is the claim address for the claim being supported.\n\nUsage:\n wallet_send ( amount | --amount= amount )\n (( address | --address= address ) | ( claim_id | --claim_id= claim_id ))\n\n\nOptions:\n --amount= amount : (float) amount of credit to send\n --address= address : (str) address to send credits to\n --claim_id= claim_id : (float) claim_id of the claim to send to tip to\n\nReturns:\n If sending to an address:\n (bool) true if payment successfully scheduled\n\n If sending a claim tip:\n (dict) Dictionary containing the result of the support\n {\n txid : (str) txid of resulting support claim\n nout : (int) nout of the resulting support claim\n fee : (float) fee paid for the transaction\n }",
"title": "wallet_send"
},
{
-23
View File
@@ -1,23 +0,0 @@
# This is an example daemon_settings.yml file.
# See https://lbry.tech/resources/daemon-settings for all configuration keys and values
share_usage_data: True
lbryum_servers:
- lbryumx1.lbry.io:50001
- lbryumx2.lbry.io:50001
blockchain_name: lbrycrd_main
data_dir: /home/lbry/.lbrynet
download_directory: /home/lbry/downloads
delete_blobs_on_remove: True
dht_node_port: 4444
peer_port: 3333
use_upnp: True
#components_to_skip:
# - peer_protocol_server
# - hash_announcer
# - dht
BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.6 KiB

+4 -2
View File
@@ -1,4 +1,6 @@
__name__ = "lbrynet"
__version__ = "0.30.5"
import logging
__version__ = "0.19.3"
version = tuple(__version__.split('.'))
logging.getLogger(__name__).addHandler(logging.NullHandler())
+268
View File
@@ -0,0 +1,268 @@
import collections
import logging
from twisted.internet import defer, task
from requests import auth
from txrequests import Session
from lbrynet import conf
from lbrynet.core import looping_call_manager, utils, system_info
# Things We Track
SERVER_STARTUP = 'Server Startup'
SERVER_STARTUP_SUCCESS = 'Server Startup Success'
SERVER_STARTUP_ERROR = 'Server Startup Error'
DOWNLOAD_STARTED = 'Download Started'
DOWNLOAD_ERRORED = 'Download Errored'
DOWNLOAD_FINISHED = 'Download Finished'
HEARTBEAT = 'Heartbeat'
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
NEW_CHANNEL = 'New Channel'
CREDITS_SENT = 'Credits Sent'
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
log = logging.getLogger(__name__)
class Manager(object):
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
self.analytics_api = analytics_api
self._tracked_data = collections.defaultdict(list)
self.looping_call_manager = self._setup_looping_calls()
self.context = context or self._make_context(
system_info.get_platform(), conf.settings['wallet'])
self.installation_id = installation_id or conf.settings.installation_id
self.session_id = session_id or conf.settings.get_session_id()
self.is_started = False
@classmethod
def new_instance(cls, enabled=None):
api = Api.new_instance(enabled)
return cls(api)
# Things We Track
def send_server_startup(self):
self.analytics_api.track(self._event(SERVER_STARTUP))
def send_server_startup_success(self):
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
def send_server_startup_error(self, message):
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
def send_download_started(self, id_, name, claim_dict=None):
self.analytics_api.track(
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
)
def send_download_errored(self, err, id_, name, claim_dict, report):
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
report)
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
def send_download_finished(self, id_, name, report, claim_dict=None):
download_properties = self._download_properties(id_, name, claim_dict, report)
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
def send_claim_action(self, action):
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
def send_new_channel(self):
self.analytics_api.track(self._event(NEW_CHANNEL))
def send_credits_sent(self):
self.analytics_api.track(self._event(CREDITS_SENT))
def _send_heartbeat(self):
self.analytics_api.track(self._event(HEARTBEAT))
def _update_tracked_metrics(self):
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
if should_send:
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
# Setup / Shutdown
def start(self):
if not self.is_started:
for name, _, interval in self._get_looping_calls():
self.looping_call_manager.start(name, interval)
self.is_started = True
def shutdown(self):
self.looping_call_manager.shutdown()
def register_repeating_metric(self, event_name, value_generator, frequency=300):
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
self.looping_call_manager.register_looping_call(event_name, lcall)
lcall.start(frequency)
def _get_looping_calls(self):
return [
('send_heartbeat', self._send_heartbeat, 60),
('update_tracked_metrics', self._update_tracked_metrics, 300),
]
def _setup_looping_calls(self):
call_manager = looping_call_manager.LoopingCallManager()
for name, fn, _ in self._get_looping_calls():
call_manager.register_looping_call(name, task.LoopingCall(fn))
return call_manager
def _send_repeating_metric(self, event_name, value_generator):
result = value_generator()
self._if_deferred(result, self._send_repeating_metric_value, event_name)
def _send_repeating_metric_value(self, result, event_name):
should_send, value = result
if should_send:
self.analytics_api.track(self._metric_event(event_name, value))
def add_observation(self, metric, value):
self._tracked_data[metric].append(value)
def summarize_and_reset(self, metric, op=sum):
"""Apply `op` on the current values for `metric`.
This operation also resets the metric.
Returns:
a tuple (should_send, value)
"""
try:
values = self._tracked_data.pop(metric)
return True, op(values)
except KeyError:
return False, None
def _event(self, event, event_properties=None):
return {
'userId': 'lbry',
'event': event,
'properties': self._event_properties(event_properties),
'context': self.context,
'timestamp': utils.isonow()
}
def _metric_event(self, metric_name, value):
return self._event(metric_name, {'value': value})
def _event_properties(self, event_properties=None):
properties = {
'lbry_id': self.installation_id,
'session_id': self.session_id,
}
properties.update(event_properties or {})
return properties
@staticmethod
def _download_properties(id_, name, claim_dict=None, report=None):
sd_hash = None if not claim_dict else claim_dict.source_hash
p = {
'download_id': id_,
'name': name,
'stream_info': sd_hash
}
if report:
p['report'] = report
return p
@staticmethod
def _download_error_properties(error, id_, name, claim_dict, report):
def error_name(err):
if not hasattr(type(err), "__name__"):
return str(type(err))
return type(err).__name__
return {
'download_id': id_,
'name': name,
'stream_info': claim_dict.source_hash,
'error': error_name(error),
'reason': error.message,
'report': report
}
@staticmethod
def _make_context(platform, wallet):
return {
'app': {
'name': 'lbrynet',
'version': platform['lbrynet_version'],
'python_version': platform['python_version'],
'build': platform['build'],
'wallet': {
'name': wallet,
'version': platform['lbryum_version'] if wallet == conf.LBRYUM_WALLET else None
},
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
'library': {
'name': 'lbrynet-analytics',
'version': '1.0.0'
},
}
@staticmethod
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
if isinstance(maybe_deferred, defer.Deferred):
maybe_deferred.addCallback(callback, *args, **kwargs)
else:
callback(maybe_deferred, *args, **kwargs)
class Api(object):
def __init__(self, session, url, write_key, enabled):
self.session = session
self.url = url
self._write_key = write_key
self._enabled = enabled
def _post(self, endpoint, data):
# there is an issue with a timing condition with keep-alive
# that is best explained here: https://github.com/mikem23/keepalive-race
#
# If you make a request, wait just the right amount of time,
# then make another request, the requests module may opt to
# reuse the connection, but by the time the server gets it the
# timeout will have expired.
#
# by forcing the connection to close, we will disable the keep-alive.
assert endpoint[0] == '/'
headers = {"Connection": "close"}
return self.session.post(
self.url + endpoint,
json=data,
auth=auth.HTTPBasicAuth(self._write_key, ''),
headers=headers
)
def track(self, event):
"""Send a single tracking event"""
if not self._enabled:
return defer.succeed('Analytics disabled')
def _log_error(failure, event):
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
log.debug('Sending track event: %s', event)
d = self._post('/track', event)
d.addErrback(_log_error, event)
return d
@classmethod
def new_instance(cls, enabled=None):
"""Initialize an instance using values from the configuration"""
session = Session()
if enabled is None:
enabled = conf.settings['share_usage_data']
return cls(
session,
conf.settings['ANALYTICS_ENDPOINT'],
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
enabled,
)
+1
View File
@@ -0,0 +1 @@
import paths
+26
View File
@@ -0,0 +1,26 @@
# App root files dir
# https://developer.android.com/reference/android/content/ContextWrapper.html#getFilesDir%28%29
def android_files_dir():
return None
# Base internal storage path
def android_internal_storage_dir():
return None
# Base external (SD card, if present) storage path
def android_external_storage_dir():
return None
# Internal device storage (private app folder)
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
def android_app_internal_storage_dir():
return None
# External (app folder on SD card, if present) storage
# https://developer.android.com/reference/android/content/ContextWrapper.html#getExternalFilesDirs(java.lang.String)
def android_app_external_storage_dir():
return None
+4
View File
@@ -0,0 +1,4 @@
from blob_file import BlobFile
from creator import BlobFileCreator
from writer import HashBlobWriter
from reader import HashBlobReader
+26 -34
View File
@@ -1,10 +1,10 @@
import os
import logging
from twisted.internet import defer
import os
from twisted.internet import defer, threads
from twisted.web.client import FileBodyProducer
from twisted.python.failure import Failure
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
from lbrynet.core.utils import is_valid_blobhash
from lbrynet.blob.writer import HashBlobWriter
from lbrynet.blob.reader import HashBlobReader
@@ -12,26 +12,8 @@ log = logging.getLogger(__name__)
MAX_BLOB_SIZE = 2 * 2 ** 20
# digest_size is in bytes, and blob hashes are hex encoded
blobhash_length = get_lbry_hash_obj().digest_size * 2
def is_valid_hashcharacter(char):
return char in "0123456789abcdef"
def is_valid_blobhash(blobhash):
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
@param blobhash: string, the blobhash to check
@return: True/False
"""
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
class BlobFile:
class BlobFile(object):
"""
A chunk of data available on the network which is specified by a hashsum
@@ -70,7 +52,7 @@ class BlobFile:
def open_for_writing(self, peer):
"""
open a blob file to be written by peer, supports concurrent
writers, as long as they are from different peers.
writers, as long as they are from differnt peers.
returns tuple of (writer, finished_deferred)
@@ -78,12 +60,12 @@ class BlobFile:
finished_deferred - deferred that is fired when write is finished and returns
a instance of itself as HashBlob
"""
if peer not in self.writers:
if not peer in self.writers:
log.debug("Opening %s to be written by %s", str(self), str(peer))
finished_deferred = defer.Deferred()
writer = HashBlobWriter(self.get_length, self.writer_finished)
self.writers[peer] = (writer, finished_deferred)
return writer, finished_deferred
return (writer, finished_deferred)
log.warning("Tried to download the same file twice simultaneously from the same peer")
return None, None
@@ -111,19 +93,29 @@ class BlobFile:
if not self.writers and not self.readers:
self._verified = False
self.saved_verified_blob = False
try:
def delete_from_file_system():
if os.path.isfile(self.file_path):
os.remove(self.file_path)
except Exception as e:
log.exception("An error occurred deleting %s:", str(self.file_path), exc_info=e)
d = threads.deferToThread(delete_from_file_system)
def log_error(err):
log.warning("An error occurred deleting %s: %s",
str(self.file_path), err.getErrorMessage())
return err
d.addErrback(log_error)
return d
else:
raise ValueError("File is currently being read or written and cannot be deleted")
return defer.fail(Failure(
ValueError("File is currently being read or written and cannot be deleted")))
@property
def verified(self):
"""
Protect verified from being modified by other classes.
verified is True if a write to a blob has completed successfully,
verified is True if a write to a blob has completed succesfully,
or a blob has been read to have the same length as specified
in init
"""
@@ -157,7 +149,7 @@ class BlobFile:
def writer_finished(self, writer, err=None):
def fire_finished_deferred():
self._verified = True
for p, (w, finished_deferred) in list(self.writers.items()):
for p, (w, finished_deferred) in self.writers.items():
if w == writer:
del self.writers[p]
finished_deferred.callback(self)
@@ -168,7 +160,7 @@ class BlobFile:
return False
def errback_finished_deferred(err):
for p, (w, finished_deferred) in list(self.writers.items()):
for p, (w, finished_deferred) in self.writers.items():
if w == writer:
del self.writers[p]
finished_deferred.errback(err)
@@ -190,7 +182,7 @@ class BlobFile:
if writer.len_so_far != self.length:
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
else:
err_string = f"blob hash is {writer.blob_hash} vs expected {self.blob_hash}"
err_string = "blob hash is %s vs expected %s" % (writer.blob_hash, self.blob_hash)
errback_finished_deferred(Failure(InvalidDataError(err_string)))
d = defer.succeed(None)
else:
+2 -2
View File
@@ -3,12 +3,12 @@ import logging
from io import BytesIO
from twisted.internet import defer
from twisted.web.client import FileBodyProducer
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbrynet.core.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
class BlobFileCreator:
class BlobFileCreator(object):
"""
This class is used to create blobs on the local filesystem
when we do not know the blob hash beforehand (i.e, when creating
+4 -2
View File
@@ -3,7 +3,7 @@ import logging
log = logging.getLogger(__name__)
class HashBlobReader:
class HashBlobReader(object):
"""
This is a file like reader class that supports
read(size) and close()
@@ -15,7 +15,7 @@ class HashBlobReader:
def __del__(self):
if self.finished_cb_d is None:
log.warning("Garbage collection was called, but reader for %s was not closed yet",
log.warn("Garbage collection was called, but reader for %s was not closed yet",
self.read_handle.name)
self.close()
@@ -28,3 +28,5 @@ class HashBlobReader:
return
self.read_handle.close()
self.finished_cb_d = self.finished_cb(self)
+5 -5
View File
@@ -1,13 +1,13 @@
import logging
from io import BytesIO
from twisted.python.failure import Failure
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbrynet.core.Error import DownloadCanceledError, InvalidDataError
from lbrynet.core.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
class HashBlobWriter:
class HashBlobWriter(object):
def __init__(self, length_getter, finished_cb):
self.write_handle = BytesIO()
self.length_getter = length_getter
@@ -18,7 +18,7 @@ class HashBlobWriter:
def __del__(self):
if self.finished_cb_d is None:
log.warning("Garbage collection was called, but writer was not closed yet")
log.warn("Garbage collection was called, but writer was not closed yet")
self.close()
@property
@@ -27,7 +27,7 @@ class HashBlobWriter:
def write(self, data):
if self.write_handle is None:
log.warning("writer has already been closed")
log.exception("writer has already been closed")
raise IOError('I/O operation on closed file')
self._hashsum.update(data)
+1 -1
View File
@@ -1,2 +1,2 @@
# don't touch this. CI server changes this during build/deployment
# dont touch this. CI server changes this during build/deployment
BUILD = "dev"
+599 -483
View File
File diff suppressed because it is too large. Load diff
@@ -1,14 +1,15 @@
import logging
import random
import time
from decimal import Decimal
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from decimal import Decimal
log = logging.getLogger(__name__)
class BlobAvailabilityTracker:
class BlobAvailabilityTracker(object):
"""
Class to track peer counts for known blobs, and to discover new popular blobs
@@ -22,14 +23,18 @@ class BlobAvailabilityTracker:
self._blob_manager = blob_manager
self._peer_finder = peer_finder
self._dht_node = dht_node
self._check_popular = LoopingCall(self._update_most_popular)
self._check_mine = LoopingCall(self._update_mine)
def start(self):
log.info("Starting blob availability tracker.")
self._check_popular.start(600)
self._check_mine.start(600)
def stop(self):
log.info("Stopping blob availability tracker.")
if self._check_popular.running:
self._check_popular.stop()
if self._check_mine.running:
self._check_mine.stop()
@@ -63,6 +68,17 @@ class BlobAvailabilityTracker:
d.addCallback(lambda peers: _save_peer_info(blob, peers))
return d
def _get_most_popular(self):
dl = []
for (hash, _) in self._dht_node.get_most_popular_hashes(10):
encoded = hash.encode('hex')
dl.append(self._update_peers_for_blob(encoded))
return defer.DeferredList(dl)
def _update_most_popular(self):
d = self._get_most_popular()
d.addCallback(lambda _: self._set_mean_peers())
def _update_mine(self):
def _get_peers(blobs):
dl = []
@@ -1,4 +1,4 @@
class BlobInfo:
class BlobInfo(object):
"""
This structure is used to represent the metadata of a blob.
@@ -16,3 +16,4 @@ class BlobInfo:
self.blob_hash = blob_hash
self.blob_num = blob_num
self.length = length
+162
View File
@@ -0,0 +1,162 @@
import logging
import os
from sqlite3 import IntegrityError
from twisted.internet import threads, defer, reactor, task
from lbrynet import conf
from lbrynet.blob.blob_file import BlobFile
from lbrynet.blob.creator import BlobFileCreator
from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier
log = logging.getLogger(__name__)
class DiskBlobManager(DHTHashSupplier):
def __init__(self, hash_announcer, blob_dir, storage):
"""
This class stores blobs on the hard disk,
blob_dir - directory where blobs are stored
db_dir - directory where sqlite database of blob information is stored
"""
DHTHashSupplier.__init__(self, hash_announcer)
self.storage = storage
self.announce_head_blobs_only = conf.settings['announce_head_blobs_only']
self.blob_dir = blob_dir
self.blob_creator_type = BlobFileCreator
# TODO: consider using an LRU for blobs as there could potentially
# be thousands of blobs loaded up, many stale
self.blobs = {}
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
self.check_should_announce_lc = None
if conf.settings['run_reflector_server']:
self.check_should_announce_lc = task.LoopingCall(self.storage.verify_will_announce_all_head_and_sd_blobs)
def setup(self):
if self.check_should_announce_lc and not self.check_should_announce_lc.running:
self.check_should_announce_lc.start(600)
return defer.succeed(True)
def stop(self):
if self.check_should_announce_lc and self.check_should_announce_lc.running:
self.check_should_announce_lc.stop()
return defer.succeed(True)
def get_blob(self, blob_hash, length=None):
"""Return a blob identified by blob_hash, which may be a new blob or a
blob that is already on the hard disk
"""
if length is not None and not isinstance(length, int):
raise Exception("invalid length type: %s (%s)" % (length, str(type(length))))
if blob_hash in self.blobs:
return defer.succeed(self.blobs[blob_hash])
return self._make_new_blob(blob_hash, length)
def get_blob_creator(self):
return self.blob_creator_type(self.blob_dir)
def _make_new_blob(self, blob_hash, length=None):
log.debug('Making a new blob for %s', blob_hash)
blob = BlobFile(self.blob_dir, blob_hash, length)
self.blobs[blob_hash] = blob
return defer.succeed(blob)
def immediate_announce(self, blob_hashes):
if self.hash_announcer:
return self.hash_announcer.immediate_announce(blob_hashes)
raise Exception("Hash announcer not set")
@defer.inlineCallbacks
def blob_completed(self, blob, next_announce_time=None, should_announce=True):
if next_announce_time is None:
next_announce_time = self.get_next_announce_time()
yield self.storage.add_completed_blob(
blob.blob_hash, blob.length, next_announce_time, should_announce
)
# we announce all blobs immediately, if announce_head_blob_only is False
# otherwise, announce only if marked as should_announce
if not self.announce_head_blobs_only or should_announce:
reactor.callLater(0, self.immediate_announce, [blob.blob_hash])
def completed_blobs(self, blobhashes_to_check):
return self._completed_blobs(blobhashes_to_check)
def hashes_to_announce(self):
return self.storage.get_blobs_to_announce(self.hash_announcer)
def count_should_announce_blobs(self):
return self.storage.count_should_announce_blobs()
def set_should_announce(self, blob_hash, should_announce):
if blob_hash in self.blobs:
blob = self.blobs[blob_hash]
if blob.get_is_verified():
return self.storage.set_should_announce(
blob_hash, self.get_next_announce_time(), should_announce
)
return defer.succeed(False)
def get_should_announce(self, blob_hash):
return self.storage.should_announce(blob_hash)
def creator_finished(self, blob_creator, should_announce):
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
if blob_creator.blob_hash is None:
raise Exception("Blob hash is None")
if blob_creator.blob_hash in self.blobs:
raise Exception("Creator finished for blob that is already marked as completed")
if blob_creator.length is None:
raise Exception("Blob has a length of 0")
new_blob = BlobFile(self.blob_dir, blob_creator.blob_hash, blob_creator.length)
self.blobs[blob_creator.blob_hash] = new_blob
next_announce_time = self.get_next_announce_time()
return self.blob_completed(new_blob, next_announce_time, should_announce)
def immediate_announce_all_blobs(self):
d = self._get_all_verified_blob_hashes()
d.addCallback(self.immediate_announce)
return d
def get_all_verified_blobs(self):
d = self._get_all_verified_blob_hashes()
d.addCallback(self.completed_blobs)
return d
@defer.inlineCallbacks
def delete_blobs(self, blob_hashes):
bh_to_delete_from_db = []
for blob_hash in blob_hashes:
try:
blob = yield self.get_blob(blob_hash)
yield blob.delete()
bh_to_delete_from_db.append(blob_hash)
del self.blobs[blob_hash]
except Exception as e:
log.warning("Failed to delete blob file. Reason: %s", e)
try:
yield self.storage.delete_blobs_from_db(bh_to_delete_from_db)
except IntegrityError as err:
if err.message != "FOREIGN KEY constraint failed":
raise err
@defer.inlineCallbacks
def _completed_blobs(self, blobhashes_to_check):
"""Returns of the blobhashes_to_check, which are valid"""
blobs = yield defer.DeferredList([self.get_blob(b) for b in blobhashes_to_check])
blob_hashes = [b.blob_hash for success, b in blobs if success and b.verified]
defer.returnValue(blob_hashes)
def _get_all_verified_blob_hashes(self):
d = self.storage.get_all_blob_hashes()
def get_verified_blobs(blobs):
verified_blobs = []
for blob_hash in blobs:
file_path = os.path.join(self.blob_dir, blob_hash)
if os.path.isfile(file_path):
verified_blobs.append(blob_hash)
return verified_blobs
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
return d
@@ -1,4 +1,4 @@
class DownloadOptionChoice:
class DownloadOptionChoice(object):
"""A possible choice that can be picked for some option.
An option can have one or more choices that can be picked from.
@@ -10,7 +10,7 @@ class DownloadOptionChoice:
self.bool_options_description = bool_options_description
class DownloadOption:
class DownloadOption(object):
"""An option for a user to select a value from several different choices."""
def __init__(self, option_types, long_description, short_description, default_value,
default_value_description):
+16 -46
View File
@@ -1,7 +1,3 @@
class RPCError(Exception):
code = 0
class PriceDisagreementError(Exception):
pass
@@ -16,19 +12,19 @@ class DownloadCanceledError(Exception):
class DownloadSDTimeout(Exception):
def __init__(self, download):
super().__init__(f'Failed to download sd blob {download} within timeout')
Exception.__init__(self, 'Failed to download sd blob {} within timeout'.format(download))
self.download = download
class DownloadTimeoutError(Exception):
def __init__(self, download):
super().__init__(f'Failed to download {download} within timeout')
Exception.__init__(self, 'Failed to download {} within timeout'.format(download))
self.download = download
class DownloadDataTimeout(Exception):
def __init__(self, download):
super().__init__('Failed to download data blobs for sd hash '
Exception.__init__(self, 'Failed to download data blobs for sd hash '
'{} within timeout'.format(download))
self.download = download
@@ -45,21 +41,7 @@ class NullFundsError(Exception):
pass
class InsufficientFundsError(RPCError):
code = -310
class CurrencyConversionError(Exception):
pass
class FileOpenError(ValueError):
# this extends ValueError because it is replacing a ValueError in EncryptedFileDownloader
# and I don't know where it might get caught upstream
pass
class ResolveError(Exception):
class InsufficientFundsError(Exception):
pass
@@ -73,41 +55,39 @@ class KeyFeeAboveMaxAllowed(Exception):
class InvalidExchangeRateResponse(Exception):
def __init__(self, source, reason):
super().__init__(f'Failed to get exchange rate from {source}:{reason}')
Exception.__init__(self, 'Failed to get exchange rate from {}:{}'.format(source, reason))
self.source = source
self.reason = reason
class UnknownNameError(Exception):
def __init__(self, name):
super().__init__(f'Name {name} is unknown')
Exception.__init__(self, 'Name {} is unknown'.format(name))
self.name = name
class UnknownClaimID(Exception):
def __init__(self, claim_id):
super().__init__(f'Claim {claim_id} is unknown')
Exception.__init__(self, 'Claim {} is unknown'.format(claim_id))
self.claim_id = claim_id
class UnknownURI(Exception):
def __init__(self, uri):
super().__init__(f'URI {uri} cannot be resolved')
Exception.__init__(self, 'URI {} cannot be resolved'.format(uri))
self.name = uri
class UnknownOutpoint(Exception):
def __init__(self, outpoint):
super().__init__(f'Outpoint {outpoint} cannot be resolved')
Exception.__init__(self, 'Outpoint {} cannot be resolved'.format(outpoint))
self.outpoint = outpoint
class InvalidName(Exception):
def __init__(self, name, invalid_characters):
self.name = name
self.invalid_characters = invalid_characters
super().__init__(
'URI contains invalid characters: {}'.format(','.join(invalid_characters)))
Exception.__init__(
self, 'URI contains invalid characters: {}'.format(','.join(invalid_characters)))
class UnknownStreamTypeError(Exception):
@@ -124,8 +104,8 @@ class InvalidStreamDescriptorError(Exception):
class InvalidStreamInfoError(Exception):
def __init__(self, name, stream_info):
msg = f'{name} has claim with invalid stream info: {stream_info}'
super().__init__(msg)
msg = '{} has claim with invalid stream info: {}'.format(name, stream_info)
Exception.__init__(self, msg)
self.name = name
self.stream_info = stream_info
@@ -175,23 +155,13 @@ class InvalidAuthenticationToken(Exception):
class NegotiationError(Exception):
pass
class InvalidCurrencyError(Exception):
def __init__(self, currency):
self.currency = currency
super().__init__(
f'Invalid currency: {currency} is not a supported currency.')
Exception.__init__(
self, 'Invalid currency: {} is not a supported currency.'.format(currency))
class NoSuchDirectoryError(Exception):
def __init__(self, directory):
self.directory = directory
super().__init__(f'No such directory {directory}')
class ComponentStartConditionNotMet(Exception):
pass
class ComponentsNotStarted(Exception):
pass
Exception.__init__(self, 'No such directory {}'.format(directory))
+18
View File
@@ -0,0 +1,18 @@
class DummyHashAnnouncer(object):
def __init__(self, *args):
pass
def run_manage_loop(self):
pass
def stop(self):
pass
def add_supplier(self, *args):
pass
def hash_queue_size(self):
return 0
def immediate_announce(self, *args):
pass
@@ -1,7 +1,7 @@
from decimal import Decimal
class Offer:
class Offer(object):
"""A rate offer to download blobs from a host."""
RATE_ACCEPTED = "RATE_ACCEPTED"
+330
View File
@@ -0,0 +1,330 @@
from collections import defaultdict
import logging
import os
import unqlite
import time
from Crypto.Hash import SHA512
from Crypto.PublicKey import RSA
from lbrynet.core.client.ClientRequest import ClientRequest
from lbrynet.core.Error import RequestCanceledError
from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, IWallet
from lbrynet.pointtraderclient import pointtraderclient
from twisted.internet import defer, threads
from zope.interface import implements
from twisted.python.failure import Failure
from lbrynet.core.Wallet import ReservedPoints
log = logging.getLogger(__name__)
class PTCWallet(object):
"""This class sends payments to peers and also ensures that expected payments are received.
This class is only intended to be used for testing."""
implements(IWallet)
def __init__(self, db_dir):
self.db_dir = db_dir
self.db = None
self.private_key = None
self.encoded_public_key = None
self.peer_pub_keys = {}
self.queued_payments = defaultdict(int)
self.expected_payments = defaultdict(list)
self.received_payments = defaultdict(list)
self.next_manage_call = None
self.payment_check_window = 3 * 60 # 3 minutes
self.new_payments_expected_time = time.time() - self.payment_check_window
self.known_transactions = []
self.total_reserved_points = 0.0
self.wallet_balance = 0.0
def manage(self):
"""Send payments, ensure expected payments are received"""
from twisted.internet import reactor
if time.time() < self.new_payments_expected_time + self.payment_check_window:
d1 = self._get_new_payments()
else:
d1 = defer.succeed(None)
d1.addCallback(lambda _: self._check_good_standing())
d2 = self._send_queued_points()
self.next_manage_call = reactor.callLater(60, self.manage)
dl = defer.DeferredList([d1, d2])
dl.addCallback(lambda _: self.get_balance())
def set_balance(balance):
self.wallet_balance = balance
dl.addCallback(set_balance)
return dl
def stop(self):
if self.next_manage_call is not None:
self.next_manage_call.cancel()
self.next_manage_call = None
d = self.manage()
self.next_manage_call.cancel()
self.next_manage_call = None
self.db = None
return d
def start(self):
def save_key(success, private_key):
if success is True:
self._save_private_key(private_key.exportKey())
return True
return False
def register_private_key(private_key):
self.private_key = private_key
self.encoded_public_key = self.private_key.publickey().exportKey()
d_r = pointtraderclient.register_new_account(private_key)
d_r.addCallback(save_key, private_key)
return d_r
def ensure_private_key_exists(encoded_private_key):
if encoded_private_key is not None:
self.private_key = RSA.importKey(encoded_private_key)
self.encoded_public_key = self.private_key.publickey().exportKey()
return True
else:
create_d = threads.deferToThread(RSA.generate, 4096)
create_d.addCallback(register_private_key)
return create_d
def start_manage():
self.manage()
return True
d = self._open_db()
d.addCallback(lambda _: self._get_wallet_private_key())
d.addCallback(ensure_private_key_exists)
d.addCallback(lambda _: start_manage())
return d
def get_info_exchanger(self):
return PointTraderKeyExchanger(self)
def get_wallet_info_query_handler_factory(self):
return PointTraderKeyQueryHandlerFactory(self)
def reserve_points(self, peer, amount):
"""Ensure a certain amount of points are available to be sent as
payment, before the service is rendered
@param peer: The peer to which the payment will ultimately be sent
@param amount: The amount of points to reserve
@return: A ReservedPoints object which is given to send_points
once the service has been rendered
"""
if self.wallet_balance >= self.total_reserved_points + amount:
self.total_reserved_points += amount
return ReservedPoints(peer, amount)
return None
def cancel_point_reservation(self, reserved_points):
"""
Return all of the points that were reserved previously for some ReservedPoints object
@param reserved_points: ReservedPoints previously returned by reserve_points
@return: None
"""
self.total_reserved_points -= reserved_points.amount
def send_points(self, reserved_points, amount):
"""
Schedule a payment to be sent to a peer
@param reserved_points: ReservedPoints object previously returned by reserve_points
@param amount: amount of points to actually send, must be less than or equal to the
amount reserved in reserved_points
@return: Deferred which fires when the payment has been scheduled
"""
self.queued_payments[reserved_points.identifier] += amount
# make any unused points available
self.total_reserved_points -= reserved_points.amount - amount
reserved_points.identifier.update_stats('points_sent', amount)
d = defer.succeed(True)
return d
def _send_queued_points(self):
ds = []
for peer, points in self.queued_payments.items():
if peer in self.peer_pub_keys:
d = pointtraderclient.send_points(
self.private_key, self.peer_pub_keys[peer], points)
self.wallet_balance -= points
self.total_reserved_points -= points
ds.append(d)
del self.queued_payments[peer]
else:
log.warning("Don't have a payment address for peer %s. Can't send %s points.",
str(peer), str(points))
return defer.DeferredList(ds)
def get_balance(self):
"""Return the balance of this wallet"""
d = pointtraderclient.get_balance(self.private_key)
return d
def add_expected_payment(self, peer, amount):
"""Increase the number of points expected to be paid by a peer"""
self.expected_payments[peer].append((amount, time.time()))
self.new_payments_expected_time = time.time()
peer.update_stats('expected_points', amount)
def set_public_key_for_peer(self, peer, pub_key):
self.peer_pub_keys[peer] = pub_key
def _get_new_payments(self):
def add_new_transactions(transactions):
for transaction in transactions:
if transaction[1] == self.encoded_public_key:
t_hash = SHA512.new()
t_hash.update(transaction[0])
t_hash.update(transaction[1])
t_hash.update(str(transaction[2]))
t_hash.update(transaction[3])
if t_hash.hexdigest() not in self.known_transactions:
self.known_transactions.append(t_hash.hexdigest())
self._add_received_payment(transaction[0], transaction[2])
d = pointtraderclient.get_recent_transactions(self.private_key)
d.addCallback(add_new_transactions)
return d
def _add_received_payment(self, encoded_other_public_key, amount):
self.received_payments[encoded_other_public_key].append((amount, time.time()))
def _check_good_standing(self):
for peer, expected_payments in self.expected_payments.iteritems():
expected_cutoff = time.time() - 90
min_expected_balance = sum([a[0] for a in expected_payments if a[1] < expected_cutoff])
received_balance = 0
if self.peer_pub_keys[peer] in self.received_payments:
received_balance = sum([
a[0] for a in self.received_payments[self.peer_pub_keys[peer]]])
if min_expected_balance > received_balance:
log.warning(
"Account in bad standing: %s (pub_key: %s), expected amount = %s, "
"received_amount = %s",
peer, self.peer_pub_keys[peer], min_expected_balance, received_balance)
def _open_db(self):
def open_db():
self.db = unqlite.UnQLite(os.path.join(self.db_dir, "ptcwallet.db"))
return threads.deferToThread(open_db)
def _save_private_key(self, private_key):
def save_key():
self.db['private_key'] = private_key
return threads.deferToThread(save_key)
def _get_wallet_private_key(self):
def get_key():
if 'private_key' in self.db:
return self.db['private_key']
return None
return threads.deferToThread(get_key)
class PointTraderKeyExchanger(object):
implements([IRequestCreator])
def __init__(self, wallet):
self.wallet = wallet
self._protocols = []
######### IRequestCreator #########
def send_next_request(self, peer, protocol):
if not protocol in self._protocols:
r = ClientRequest({'public_key': self.wallet.encoded_public_key},
'public_key')
d = protocol.add_request(r)
d.addCallback(self._handle_exchange_response, peer, r, protocol)
d.addErrback(self._request_failed, peer)
self._protocols.append(protocol)
return defer.succeed(True)
else:
return defer.succeed(False)
######### internal calls #########
def _handle_exchange_response(self, response_dict, peer, request, protocol):
assert request.response_identifier in response_dict, \
"Expected %s in dict but did not get it" % request.response_identifier
assert protocol in self._protocols, "Responding protocol is not in our list of protocols"
peer_pub_key = response_dict[request.response_identifier]
self.wallet.set_public_key_for_peer(peer, peer_pub_key)
return True
def _request_failed(self, err, peer):
if not err.check(RequestCanceledError):
log.warning("A peer failed to send a valid public key response. Error: %s, peer: %s",
err.getErrorMessage(), str(peer))
return err
class PointTraderKeyQueryHandlerFactory(object):
implements(IQueryHandlerFactory)
def __init__(self, wallet):
self.wallet = wallet
######### IQueryHandlerFactory #########
def build_query_handler(self):
q_h = PointTraderKeyQueryHandler(self.wallet)
return q_h
def get_primary_query_identifier(self):
return 'public_key'
def get_description(self):
return ("Point Trader Address - an address for receiving payments on the "
"point trader testing network")
class PointTraderKeyQueryHandler(object):
implements(IQueryHandler)
def __init__(self, wallet):
self.wallet = wallet
self.query_identifiers = ['public_key']
self.public_key = None
self.peer = None
######### IQueryHandler #########
def register_with_request_handler(self, request_handler, peer):
self.peer = peer
request_handler.register_query_handler(self, self.query_identifiers)
def handle_queries(self, queries):
if self.query_identifiers[0] in queries:
new_encoded_pub_key = queries[self.query_identifiers[0]]
try:
RSA.importKey(new_encoded_pub_key)
except (ValueError, TypeError, IndexError):
log.warning("Client sent an invalid public key.")
return defer.fail(Failure(ValueError("Client sent an invalid public key")))
self.public_key = new_encoded_pub_key
self.wallet.set_public_key_for_peer(self.peer, self.public_key)
log.debug("Received the client's public key: %s", str(self.public_key))
fields = {'public_key': self.wallet.encoded_public_key}
return defer.succeed(fields)
if self.public_key is None:
log.warning("Expected a public key, but did not receive one")
return defer.fail(Failure(ValueError("Expected but did not receive a public key")))
else:
return defer.succeed({})
@@ -1,14 +1,16 @@
from lbrynet.core.Strategy import get_default_strategy, OnlyFreeStrategy
from lbrynet import conf
from decimal import Decimal
from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy
class BasePaymentRateManager:
def __init__(self, rate, info_rate):
self.min_blob_data_payment_rate = rate
self.min_blob_info_payment_rate = info_rate
class BasePaymentRateManager(object):
def __init__(self, rate=None, info_rate=None):
self.min_blob_data_payment_rate = rate if rate is not None else conf.settings['data_rate']
self.min_blob_info_payment_rate = (
info_rate if info_rate is not None else conf.settings['min_info_rate'])
class PaymentRateManager:
class PaymentRateManager(object):
def __init__(self, base, rate=None):
"""
@param base: a BasePaymentRateManager
@@ -34,8 +36,8 @@ class PaymentRateManager:
self.points_paid += amount
class NegotiatedPaymentRateManager:
def __init__(self, base, availability_tracker, generous):
class NegotiatedPaymentRateManager(object):
def __init__(self, base, availability_tracker, generous=None):
"""
@param base: a BasePaymentRateManager
@param availability_tracker: a BlobAvailabilityTracker
@@ -46,10 +48,10 @@ class NegotiatedPaymentRateManager:
self.min_blob_data_payment_rate = base.min_blob_data_payment_rate
self.points_paid = 0.0
self.blob_tracker = availability_tracker
self.generous = generous
self.strategy = get_default_strategy(
self.blob_tracker, self.base.min_blob_data_payment_rate, generous
)
self.generous = generous if generous is not None else conf.settings['is_generous_host']
self.strategy = get_default_strategy(self.blob_tracker,
base_price=self.base.min_blob_data_payment_rate,
is_generous=generous)
def get_rate_blob_data(self, peer, blobs):
response = self.strategy.make_offer(peer, blobs)
@@ -82,7 +84,7 @@ class NegotiatedPaymentRateManager:
return False
class OnlyFreePaymentsManager:
class OnlyFreePaymentsManager(object):
def __init__(self, **kwargs):
"""
A payment rate manager that will only ever accept and offer a rate of 0.0,
@@ -91,7 +93,6 @@ class OnlyFreePaymentsManager:
self.base = BasePaymentRateManager(0.0, 0.0)
self.points_paid = 0.0
self.min_blob_data_payment_rate = 0.0
self.generous = True
self.strategy = OnlyFreeStrategy()
+5 -5
View File
@@ -1,9 +1,9 @@
import datetime
from collections import defaultdict
from lbrynet import utils
from lbrynet.core import utils
# Do not create this object except through PeerManager
class Peer:
class Peer(object):
def __init__(self, host, port):
self.host = host
self.port = port
@@ -11,7 +11,7 @@ class Peer:
self.attempt_connection_at = None
# Number of times this Peer has been reported to be down, resets to 0 when it is up
self.down_count = 0
# Number of successful connections (with full protocol completion) to this peer
# Number of succesful connections (with full protocol completion) to this peer
self.success_count = 0
self.score = 0
self.stats = defaultdict(float) # {string stat_type, float count}
@@ -40,7 +40,7 @@ class Peer:
self.stats[stat_type] += count
def __str__(self):
return f'{self.host}:{self.port}'
return '{}:{}'.format(self.host, self.port)
def __repr__(self):
return f'Peer({self.host!r}, {self.port!r})'
return 'Peer({!r}, {!r})'.format(self.host, self.port)
+19
View File
@@ -0,0 +1,19 @@
from twisted.internet import defer
class DummyPeerFinder(object):
"""This class finds peers which have announced to the DHT that they have certain blobs"""
def __init__(self):
pass
def run_manage_loop(self):
pass
def stop(self):
pass
def find_peers_for_blob(self, blob_hash):
return defer.succeed([])
def get_most_popular_hashes(self, num_to_return):
return []
@@ -1,7 +1,7 @@
from lbrynet.p2p.Peer import Peer
from lbrynet.core.Peer import Peer
class PeerManager:
class PeerManager(object):
def __init__(self):
self.peers = []
@@ -1,11 +1,15 @@
from zope.interface import implementer
from decimal import Decimal
from lbrynet.interfaces import IBlobPriceModel
from lbrynet import conf
def get_default_price_model(blob_tracker, base_price, **kwargs):
return MeanAvailabilityWeightedPrice(blob_tracker, base_price, **kwargs)
class ZeroPrice:
class ZeroPrice(object):
def __init__(self):
self.base_price = 0.0
@@ -13,7 +17,7 @@ class ZeroPrice:
return 0.0
class MeanAvailabilityWeightedPrice:
class MeanAvailabilityWeightedPrice(object):
"""Calculate mean-blob-availability and stream-position weighted price for a blob
Attributes:
@@ -24,9 +28,11 @@ class MeanAvailabilityWeightedPrice:
blob_tracker (BlobAvailabilityTracker): blob availability tracker
"""
implementer(IBlobPriceModel)
def __init__(self, tracker, base_price, alpha=1.0):
def __init__(self, tracker, base_price=None, alpha=1.0):
self.blob_tracker = tracker
base_price = base_price if base_price is not None else conf.settings['data_rate']
self.base_price = Decimal(base_price)
self.alpha = Decimal(alpha)
@@ -1,12 +1,14 @@
import logging
from zope.interface import implements
from lbrynet.interfaces import IRateLimiter
from twisted.internet import task
log = logging.getLogger(__name__)
class DummyRateLimiter:
class DummyRateLimiter(object):
def __init__(self):
self.dl_bytes_this_second = 0
self.ul_bytes_this_second = 0
@@ -44,10 +46,10 @@ class DummyRateLimiter:
self.total_ul_bytes += num_bytes
class RateLimiter:
class RateLimiter(object):
"""This class ensures that upload and download rates don't exceed specified maximums"""
#implements(IRateLimiter)
implements(IRateLimiter)
#called by main application
+359
View File
@@ -0,0 +1,359 @@
import logging
import miniupnpc
from lbrynet.core.BlobManager import DiskBlobManager
from lbrynet.dht import node
from lbrynet.database.storage import SQLiteStorage
from lbrynet.core.PeerManager import PeerManager
from lbrynet.core.RateLimiter import RateLimiter
from lbrynet.core.client.DHTPeerFinder import DHTPeerFinder
from lbrynet.core.HashAnnouncer import DummyHashAnnouncer
from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer
from lbrynet.core.utils import generate_id
from lbrynet.core.PaymentRateManager import BasePaymentRateManager, NegotiatedPaymentRateManager
from lbrynet.core.BlobAvailability import BlobAvailabilityTracker
from twisted.internet import threads, defer
log = logging.getLogger(__name__)
class Session(object):
"""This class manages all important services common to any application that uses the network.
the hash announcer, which informs other peers that this peer is
associated with some hash. Usually, this means this peer has a
blob identified by the hash in question, but it can be used for
other purposes.
the peer finder, which finds peers that are associated with some
hash.
the blob manager, which keeps track of which blobs have been
downloaded and provides access to them,
the rate limiter, which attempts to ensure download and upload
rates stay below a set maximum
upnp, which opens holes in compatible firewalls so that remote
peers can connect to this peer.
"""
def __init__(self, blob_data_payment_rate, db_dir=None,
node_id=None, peer_manager=None, dht_node_port=None,
known_dht_nodes=None, peer_finder=None,
hash_announcer=None, blob_dir=None,
blob_manager=None, peer_port=None, use_upnp=True,
rate_limiter=None, wallet=None,
dht_node_class=node.Node, blob_tracker_class=None,
payment_rate_manager_class=None, is_generous=True, external_ip=None, storage=None):
"""@param blob_data_payment_rate: The default payment rate for blob data
@param db_dir: The directory in which levelDB files should be stored
@param node_id: The unique ID of this node
@param peer_manager: An object which keeps track of all known
peers. If None, a PeerManager will be created
@param dht_node_port: The port on which the dht node should
listen for incoming connections
@param known_dht_nodes: A list of nodes which the dht node
should use to bootstrap into the dht
@param peer_finder: An object which is used to look up peers
that are associated with some hash. If None, a
DHTPeerFinder will be used, which looks for peers in the
distributed hash table.
@param hash_announcer: An object which announces to other
peers that this peer is associated with some hash. If
None, and peer_port is not None, a DHTHashAnnouncer will
be used. If None and peer_port is None, a
DummyHashAnnouncer will be used, which will not actually
announce anything.
@param blob_dir: The directory in which blobs will be
stored. If None and blob_manager is None, blobs will be
stored in memory only.
@param blob_manager: An object which keeps track of downloaded
blobs and provides access to them. If None, and blob_dir
is not None, a DiskBlobManager will be used, with the
given blob_dir. If None and blob_dir is None, a
TempBlobManager will be used, which stores blobs in memory
only.
@param peer_port: The port on which other peers should connect
to this peer
@param use_upnp: Whether or not to try to open a hole in the
firewall so that outside peers can connect to this peer's
peer_port and dht_node_port
@param rate_limiter: An object which keeps track of the amount
of data transferred to and from this peer, and can limit
that rate if desired
@param wallet: An object which will be used to keep track of
expected payments and which will pay peers. If None, a
wallet which uses the Point Trader system will be used,
which is meant for testing only
"""
self.db_dir = db_dir
self.node_id = node_id
self.peer_manager = peer_manager
self.dht_node_port = dht_node_port
self.known_dht_nodes = known_dht_nodes
if self.known_dht_nodes is None:
self.known_dht_nodes = []
self.peer_finder = peer_finder
self.hash_announcer = hash_announcer
self.blob_dir = blob_dir
self.blob_manager = blob_manager
self.blob_tracker = None
self.blob_tracker_class = blob_tracker_class or BlobAvailabilityTracker
self.peer_port = peer_port
self.use_upnp = use_upnp
self.rate_limiter = rate_limiter
self.external_ip = external_ip
self.upnp_redirects = []
self.wallet = wallet
self.dht_node_class = dht_node_class
self.dht_node = None
self.base_payment_rate_manager = BasePaymentRateManager(blob_data_payment_rate)
self.payment_rate_manager = None
self.payment_rate_manager_class = payment_rate_manager_class or NegotiatedPaymentRateManager
self.is_generous = is_generous
self.storage = storage or SQLiteStorage(self.db_dir)
def setup(self):
"""Create the blob directory and database if necessary, start all desired services"""
log.debug("Starting session.")
if self.node_id is None:
self.node_id = generate_id()
if self.wallet is None:
from lbrynet.core.PTCWallet import PTCWallet
self.wallet = PTCWallet(self.db_dir)
if self.peer_manager is None:
self.peer_manager = PeerManager()
if self.use_upnp is True:
d = self._try_upnp()
else:
d = defer.succeed(True)
if self.peer_finder is None:
d.addCallback(lambda _: self._setup_dht())
else:
if self.hash_announcer is None and self.peer_port is not None:
log.warning("The server has no way to advertise its available blobs.")
self.hash_announcer = DummyHashAnnouncer()
d.addCallback(lambda _: self._setup_other_components())
return d
def shut_down(self):
"""Stop all services"""
log.info('Stopping session.')
ds = []
if self.blob_tracker is not None:
ds.append(defer.maybeDeferred(self.blob_tracker.stop))
if self.dht_node is not None:
ds.append(defer.maybeDeferred(self.dht_node.stop))
if self.rate_limiter is not None:
ds.append(defer.maybeDeferred(self.rate_limiter.stop))
if self.peer_finder is not None:
ds.append(defer.maybeDeferred(self.peer_finder.stop))
if self.hash_announcer is not None:
ds.append(defer.maybeDeferred(self.hash_announcer.stop))
if self.wallet is not None:
ds.append(defer.maybeDeferred(self.wallet.stop))
if self.blob_manager is not None:
ds.append(defer.maybeDeferred(self.blob_manager.stop))
if self.use_upnp is True:
ds.append(defer.maybeDeferred(self._unset_upnp))
return defer.DeferredList(ds)
def _try_upnp(self):
log.debug("In _try_upnp")
def get_free_port(upnp, port, protocol):
# returns an existing mapping if it exists
mapping = upnp.getspecificportmapping(port, protocol)
if not mapping:
return port
if upnp.lanaddr == mapping[0]:
return mapping
return get_free_port(upnp, port + 1, protocol)
def get_port_mapping(upnp, internal_port, protocol, description):
# try to map to the requested port, if there is already a mapping use the next external
# port available
if protocol not in ['UDP', 'TCP']:
raise Exception("invalid protocol")
external_port = get_free_port(upnp, internal_port, protocol)
if isinstance(external_port, tuple):
log.info("Found existing UPnP redirect %s:%i (%s) to %s:%i, using it",
self.external_ip, external_port[1], protocol, upnp.lanaddr, internal_port)
return external_port[1], protocol
upnp.addportmapping(external_port, protocol, upnp.lanaddr, internal_port,
description, '')
log.info("Set UPnP redirect %s:%i (%s) to %s:%i", self.external_ip, external_port,
protocol, upnp.lanaddr, internal_port)
return external_port, protocol
def threaded_try_upnp():
if self.use_upnp is False:
log.debug("Not using upnp")
return False
u = miniupnpc.UPnP()
num_devices_found = u.discover()
if num_devices_found > 0:
u.selectigd()
external_ip = u.externalipaddress()
if external_ip != '0.0.0.0' and not self.external_ip:
# best not to rely on this external ip, the router can be behind layers of NATs
self.external_ip = external_ip
if self.peer_port:
self.upnp_redirects.append(
get_port_mapping(u, self.peer_port, 'TCP', 'LBRY peer port')
)
if self.dht_node_port:
self.upnp_redirects.append(
get_port_mapping(u, self.dht_node_port, 'UDP', 'LBRY DHT port')
)
return True
return False
def upnp_failed(err):
log.warning("UPnP failed. Reason: %s", err.getErrorMessage())
return False
d = threads.deferToThread(threaded_try_upnp)
d.addErrback(upnp_failed)
return d
# the callback, if any, will be invoked once the joining procedure
# has terminated
def join_dht(self, cb=None):
from twisted.internet import reactor
def join_resolved_addresses(result):
addresses = []
for success, value in result:
if success is True:
addresses.append(value)
return addresses
@defer.inlineCallbacks
def join_network(knownNodes):
log.debug("join DHT using known nodes: " + str(knownNodes))
result = yield self.dht_node.joinNetwork(knownNodes)
defer.returnValue(result)
ds = []
for host, port in self.known_dht_nodes:
d = reactor.resolve(host)
d.addCallback(lambda h: (h, port)) # match host to port
ds.append(d)
dl = defer.DeferredList(ds)
dl.addCallback(join_resolved_addresses)
dl.addCallback(join_network)
if cb:
dl.addCallback(cb)
return dl
def _setup_dht(self):
log.info("Starting DHT")
def start_dht(join_network_result):
self.hash_announcer.run_manage_loop()
return True
self.dht_node = self.dht_node_class(
udpPort=self.dht_node_port,
node_id=self.node_id,
externalIP=self.external_ip,
peerPort=self.peer_port
)
self.peer_finder = DHTPeerFinder(self.dht_node, self.peer_manager)
if self.hash_announcer is None:
self.hash_announcer = DHTHashAnnouncer(self.dht_node, self.peer_port)
self.dht_node.startNetwork()
# pass start_dht() as callback to start the remaining components after joining the DHT
return self.join_dht(start_dht)
def _setup_other_components(self):
log.debug("Setting up the rest of the components")
if self.rate_limiter is None:
self.rate_limiter = RateLimiter()
if self.blob_manager is None:
if self.blob_dir is None:
raise Exception(
"TempBlobManager is no longer supported, specify BlobManager or db_dir")
else:
self.blob_manager = DiskBlobManager(
self.hash_announcer, self.blob_dir, self.storage
)
if self.blob_tracker is None:
self.blob_tracker = self.blob_tracker_class(
self.blob_manager, self.peer_finder, self.dht_node
)
if self.payment_rate_manager is None:
self.payment_rate_manager = self.payment_rate_manager_class(
self.base_payment_rate_manager, self.blob_tracker, self.is_generous
)
self.rate_limiter.start()
d = self.storage.setup()
d.addCallback(lambda _: self.blob_manager.setup())
d.addCallback(lambda _: self.wallet.start())
d.addCallback(lambda _: self.blob_tracker.start())
return d
def _unset_upnp(self):
log.info("Unsetting upnp for session")
def threaded_unset_upnp():
u = miniupnpc.UPnP()
num_devices_found = u.discover()
if num_devices_found > 0:
u.selectigd()
for port, protocol in self.upnp_redirects:
if u.getspecificportmapping(port, protocol) is None:
log.warning(
"UPnP redirect for %s %d was removed by something else.",
protocol, port)
else:
u.deleteportmapping(port, protocol)
log.info("Removed UPnP redirect for %s %d.", protocol, port)
self.upnp_redirects = []
d = threads.deferToThread(threaded_unset_upnp)
d.addErrback(lambda err: str(err))
return d
@@ -2,27 +2,24 @@ import logging
import shutil
import tempfile
from twisted.internet import defer, reactor
from lbrynet.extras.compat import d2f
from lbrynet.blob.blob_file import BlobFile
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.p2p.RateLimiter import DummyRateLimiter
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
from lbrynet.p2p.client.BlobRequester import BlobRequester
from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder
from lbrynet.conf import Config
from twisted.internet import defer, threads, reactor
from lbrynet.blob import BlobFile
from lbrynet.core.BlobManager import DiskBlobManager
from lbrynet.core.HashAnnouncer import DummyHashAnnouncer
from lbrynet.core.RateLimiter import DummyRateLimiter
from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager
from lbrynet.core.PeerFinder import DummyPeerFinder
from lbrynet.core.client.BlobRequester import BlobRequester
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.core.client.ConnectionManager import ConnectionManager
log = logging.getLogger(__name__)
class SinglePeerFinder(DummyPeerFinder):
def __init__(self, peer):
super().__init__()
DummyPeerFinder.__init__(self)
self.peer = peer
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=False):
@@ -31,7 +28,7 @@ class SinglePeerFinder(DummyPeerFinder):
class BlobCallback(BlobFile):
def __init__(self, blob_dir, blob_hash, timeout):
super().__init__(blob_dir, blob_hash)
BlobFile.__init__(self, blob_dir, blob_hash)
self.callback = defer.Deferred()
reactor.callLater(timeout, self._cancel)
@@ -46,7 +43,7 @@ class BlobCallback(BlobFile):
return result
class SingleBlobDownloadManager:
class SingleBlobDownloadManager(object):
def __init__(self, blob):
self.blob = blob
@@ -60,10 +57,10 @@ class SingleBlobDownloadManager:
return self.blob.blob_hash
class SinglePeerDownloader:
def __init__(self, conf: Config):
self.conf = conf
class SinglePeerDownloader(object):
def __init__(self):
self._payment_rate_manager = OnlyFreePaymentsManager()
self._announcer = DummyHashAnnouncer()
self._rate_limiter = DummyRateLimiter()
self._wallet = None
self._blob_manager = None
@@ -83,7 +80,7 @@ class SinglePeerDownloader:
peer_finder = SinglePeerFinder(peer)
requester = BlobRequester(blob_manager, peer_finder, self._payment_rate_manager,
self._wallet, download_manager)
downloader = StandaloneBlobDownloader(self.conf, blob_hash, blob_manager, peer_finder,
downloader = StandaloneBlobDownloader(blob_hash, blob_manager, peer_finder,
self._rate_limiter, self._payment_rate_manager,
self._wallet, timeout=timeout)
info_exchanger = self._wallet.get_info_exchanger()
@@ -94,16 +91,16 @@ class SinglePeerDownloader:
result = yield blob.callback
if not result:
log.debug("Failed to downloaded %s from %s", blob_hash[:16], peer.host)
yield connection_manager.stop()
yield connection_manager.stop()
defer.returnValue(result)
async def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
tmp_storage = SQLiteStorage(Config(), ':memory:')
await tmp_storage.open()
tmp_dir = tempfile.mkdtemp()
tmp_blob_manager = DiskBlobManager(tmp_dir, tmp_storage)
@defer.inlineCallbacks
def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
tmp_blob_manager = DiskBlobManager(self._announcer, tmp_dir, tmp_dir)
try:
return await d2f(self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager))
result = yield self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager)
finally:
await tmp_storage.close()
shutil.rmtree(tmp_dir)
yield tmp_blob_manager.stop()
yield threads.deferToThread(shutil.rmtree, tmp_dir)
defer.returnValue(result)
@@ -1,20 +1,25 @@
from zope.interface import implementer
from decimal import Decimal
from lbrynet.p2p.Offer import Offer
from lbrynet.p2p.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
from lbrynet import conf
from lbrynet.interfaces import INegotiationStrategy
from lbrynet.core.Offer import Offer
from lbrynet.core.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
def get_default_strategy(blob_tracker, base_price, is_generous, **kwargs):
return BasicAvailabilityWeightedStrategy(blob_tracker, base_price, is_generous, **kwargs)
def get_default_strategy(blob_tracker, **kwargs):
return BasicAvailabilityWeightedStrategy(blob_tracker, **kwargs)
class Strategy:
class Strategy(object):
"""
Base for negotiation strategies
"""
implementer(INegotiationStrategy)
def __init__(self, price_model, max_rate, min_rate, is_generous):
def __init__(self, price_model, max_rate, min_rate, is_generous=None):
self.price_model = price_model
self.is_generous = is_generous
self.is_generous = (
is_generous if is_generous is not None else conf.settings['is_generous_host'])
self.accepted_offers = {}
self.pending_sent_offers = {}
self.offers_sent = {}
@@ -95,13 +100,16 @@ class BasicAvailabilityWeightedStrategy(Strategy):
accepted or a threshold is reached
"""
implementer(INegotiationStrategy)
def __init__(self, blob_tracker, base_price, is_generous,
acceleration=1.25, deceleration=0.9, max_rate=None,
min_rate=0.0, alpha=1.0):
def __init__(self, blob_tracker, acceleration=1.25,
deceleration=0.9, max_rate=None,
min_rate=0.0,
is_generous=None,
base_price=0.0001, alpha=1.0):
price_model = MeanAvailabilityWeightedPrice(
blob_tracker, base_price, alpha=alpha)
super().__init__(price_model, max_rate, min_rate, is_generous)
blob_tracker, base_price=base_price, alpha=alpha)
Strategy.__init__(self, price_model, max_rate, min_rate, is_generous)
self._acceleration = Decimal(acceleration) # rate of how quickly to ramp offer
self._deceleration = Decimal(deceleration)
@@ -129,9 +137,10 @@ class BasicAvailabilityWeightedStrategy(Strategy):
class OnlyFreeStrategy(Strategy):
implementer(INegotiationStrategy)
def __init__(self, *args, **kwargs):
price_model = ZeroPrice()
super().__init__(price_model, 0.0, 0.0, True)
Strategy.__init__(self, price_model, 0.0, 0.0, True)
def _get_mean_rate(self, rates):
return 0.0
@@ -1,27 +1,18 @@
import string
import binascii
from collections import defaultdict
import json
import logging
from collections import defaultdict
from binascii import unhexlify
from twisted.internet import threads, defer
from lbrynet.extras.compat import f2d
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.p2p.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
from twisted.internet import threads, defer
from lbrynet.core.cryptoutils import get_lbry_hash_obj
from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.core.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
log = logging.getLogger(__name__)
class JSONBytesEncoder(json.JSONEncoder):
def default(self, obj): # pylint: disable=E0202
if isinstance(obj, bytes):
return obj.decode()
return super().default(obj)
class StreamDescriptorReader:
class StreamDescriptorReader(object):
"""Classes which derive from this class read a stream descriptor file return
a dictionary containing the fields in the file"""
def __init__(self):
@@ -30,6 +21,7 @@ class StreamDescriptorReader:
def _get_raw_data(self):
"""This method must be overridden by subclasses. It should return a deferred
which fires with the raw data in the stream descriptor"""
pass
def get_info(self):
"""Return the fields contained in the file"""
@@ -41,7 +33,7 @@ class StreamDescriptorReader:
class PlainStreamDescriptorReader(StreamDescriptorReader):
"""Read a stream descriptor file which is not a blob but a regular file"""
def __init__(self, stream_descriptor_filename):
super().__init__()
StreamDescriptorReader.__init__(self)
self.stream_descriptor_filename = stream_descriptor_filename
def _get_raw_data(self):
@@ -57,7 +49,7 @@ class PlainStreamDescriptorReader(StreamDescriptorReader):
class BlobStreamDescriptorReader(StreamDescriptorReader):
"""Read a stream descriptor file which is a blob"""
def __init__(self, blob):
super().__init__()
StreamDescriptorReader.__init__(self)
self.blob = blob
def _get_raw_data(self):
@@ -74,26 +66,25 @@ class BlobStreamDescriptorReader(StreamDescriptorReader):
return threads.deferToThread(get_data)
class StreamDescriptorWriter:
class StreamDescriptorWriter(object):
"""Classes which derive from this class write fields from a dictionary
of fields to a stream descriptor"""
def __init__(self):
pass
def create_descriptor(self, sd_info):
return self._write_stream_descriptor(
json.dumps(sd_info, sort_keys=True).encode()
)
return self._write_stream_descriptor(json.dumps(sd_info))
def _write_stream_descriptor(self, raw_data):
"""This method must be overridden by subclasses to write raw data to
the stream descriptor
"""
pass
class PlainStreamDescriptorWriter(StreamDescriptorWriter):
def __init__(self, sd_file_name):
super().__init__()
StreamDescriptorWriter.__init__(self)
self.sd_file_name = sd_file_name
def _write_stream_descriptor(self, raw_data):
@@ -109,7 +100,7 @@ class PlainStreamDescriptorWriter(StreamDescriptorWriter):
class BlobStreamDescriptorWriter(StreamDescriptorWriter):
def __init__(self, blob_manager):
super().__init__()
StreamDescriptorWriter.__init__(self)
self.blob_manager = blob_manager
@defer.inlineCallbacks
@@ -123,7 +114,7 @@ class BlobStreamDescriptorWriter(StreamDescriptorWriter):
defer.returnValue(sd_hash)
class StreamMetadata:
class StreamMetadata(object):
FROM_BLOB = 1
FROM_PLAIN = 2
@@ -136,7 +127,7 @@ class StreamMetadata:
self.source_file = None
class StreamDescriptorIdentifier:
class StreamDescriptorIdentifier(object):
"""Tries to determine the type of stream described by the stream descriptor using the
'stream_type' field. Keeps a list of StreamDescriptorValidators and StreamDownloaderFactorys
and returns the appropriate ones based on the type of the stream descriptor given
@@ -261,18 +252,18 @@ def save_sd_info(blob_manager, sd_hash, sd_info):
if calculated_sd_hash != sd_hash:
raise InvalidStreamDescriptorError("%s does not match calculated %s" %
(sd_hash, calculated_sd_hash))
stream_hash = yield f2d(blob_manager.storage.get_stream_hash_for_sd_hash(sd_hash))
stream_hash = yield blob_manager.storage.get_stream_hash_for_sd_hash(sd_hash)
if not stream_hash:
log.debug("Saving info for %s", unhexlify(sd_info['stream_name']))
log.debug("Saving info for %s", sd_info['stream_name'].decode('hex'))
stream_name = sd_info['stream_name']
key = sd_info['key']
stream_hash = sd_info['stream_hash']
stream_blobs = sd_info['blobs']
suggested_file_name = sd_info['suggested_file_name']
yield f2d(blob_manager.storage.add_known_blobs(stream_blobs))
yield f2d(blob_manager.storage.store_stream(
yield blob_manager.storage.add_known_blobs(stream_blobs)
yield blob_manager.storage.store_stream(
stream_hash, sd_hash, stream_name, key, suggested_file_name, stream_blobs
))
)
defer.returnValue(stream_hash)
@@ -281,9 +272,9 @@ def format_blobs(crypt_blob_infos):
for blob_info in crypt_blob_infos:
blob = {}
if blob_info.length != 0:
blob['blob_hash'] = blob_info.blob_hash
blob['blob_hash'] = str(blob_info.blob_hash)
blob['blob_num'] = blob_info.blob_num
blob['iv'] = blob_info.iv
blob['iv'] = str(blob_info.iv)
blob['length'] = blob_info.length
formatted_blobs.append(blob)
return formatted_blobs
@@ -300,7 +291,8 @@ def format_sd_info(stream_type, stream_name, key, suggested_file_name, stream_ha
}
async def get_sd_info(storage, stream_hash, include_blobs):
@defer.inlineCallbacks
def get_sd_info(storage, stream_hash, include_blobs):
"""
Get an sd info dictionary from storage
@@ -329,13 +321,16 @@ async def get_sd_info(storage, stream_hash, include_blobs):
]
}
"""
stream_info = await storage.get_stream_info(stream_hash)
stream_info = yield storage.get_stream_info(stream_hash)
blobs = []
if include_blobs:
blobs = await storage.get_blobs_for_stream(stream_hash)
return format_sd_info(
EncryptedFileStreamType, stream_info[0], stream_info[1],
stream_info[2], stream_hash, format_blobs(blobs)
blobs = yield storage.get_blobs_for_stream(stream_hash)
defer.returnValue(
format_sd_info(
EncryptedFileStreamType, stream_info[0], stream_info[1],
stream_info[2], stream_hash, format_blobs(blobs)
)
)
@@ -349,18 +344,18 @@ def get_blob_hashsum(b):
iv = b['iv']
blob_hashsum = get_lbry_hash_obj()
if length != 0:
blob_hashsum.update(blob_hash.encode())
blob_hashsum.update(str(blob_num).encode())
blob_hashsum.update(iv.encode())
blob_hashsum.update(str(length).encode())
blob_hashsum.update(blob_hash)
blob_hashsum.update(str(blob_num))
blob_hashsum.update(iv)
blob_hashsum.update(str(length))
return blob_hashsum.digest()
def get_stream_hash(hex_stream_name, key, hex_suggested_file_name, blob_infos):
h = get_lbry_hash_obj()
h.update(hex_stream_name.encode())
h.update(key.encode())
h.update(hex_suggested_file_name.encode())
h.update(hex_stream_name)
h.update(key)
h.update(hex_suggested_file_name)
blobs_hashsum = get_lbry_hash_obj()
for blob in blob_infos:
blobs_hashsum.update(get_blob_hashsum(blob))
@@ -369,8 +364,9 @@ def get_stream_hash(hex_stream_name, key, hex_suggested_file_name, blob_infos):
def verify_hex(text, field_name):
if not set(text).issubset(set(string.hexdigits)):
raise InvalidStreamDescriptorError("%s is not a hex-encoded string" % field_name)
for c in text:
if c not in '0123456789abcdef':
raise InvalidStreamDescriptorError("%s is not a hex-encoded string" % field_name)
def validate_descriptor(stream_info):
@@ -384,7 +380,7 @@ def validate_descriptor(stream_info):
raise InvalidStreamDescriptorError("Missing '%s'" % (e.args[0]))
if stream_info['blobs'][-1]['length'] != 0:
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
if any([blob_info['length'] == 0 for blob_info in stream_info['blobs'][:-1]]):
if any([False if blob_info['length'] > 0 else True for blob_info in stream_info['blobs'][:-1]]):
raise InvalidStreamDescriptorError("Contains zero-length data blob")
if 'blob_hash' in stream_info['blobs'][-1]:
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
@@ -401,7 +397,7 @@ def validate_descriptor(stream_info):
return True
class EncryptedFileStreamDescriptorValidator:
class EncryptedFileStreamDescriptorValidator(object):
def __init__(self, raw_info):
self.raw_info = raw_info
@@ -410,14 +406,14 @@ class EncryptedFileStreamDescriptorValidator:
def info_to_show(self):
info = []
info.append(("stream_name", unhexlify(self.raw_info.get("stream_name"))))
info.append(("stream_name", binascii.unhexlify(self.raw_info.get("stream_name"))))
size_so_far = 0
for blob_info in self.raw_info.get("blobs", []):
size_so_far += int(blob_info['length'])
info.append(("stream_size", str(self.get_length_of_stream())))
suggested_file_name = self.raw_info.get("suggested_file_name", None)
if suggested_file_name is not None:
suggested_file_name = unhexlify(suggested_file_name)
suggested_file_name = binascii.unhexlify(suggested_file_name)
info.append(("suggested_file_name", suggested_file_name))
return info
@@ -429,8 +425,7 @@ class EncryptedFileStreamDescriptorValidator:
@defer.inlineCallbacks
def download_sd_blob(conf, blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet,
timeout=None, download_mirrors=None):
def download_sd_blob(session, blob_hash, payment_rate_manager, timeout=None):
"""
Downloads a single blob from the network
@@ -443,26 +438,22 @@ def download_sd_blob(conf, blob_hash, blob_manager, peer_finder, rate_limiter, p
@return: An object of type HashBlob
"""
downloader = StandaloneBlobDownloader(conf,
blob_hash,
blob_manager,
peer_finder,
rate_limiter,
downloader = StandaloneBlobDownloader(blob_hash,
session.blob_manager,
session.peer_finder,
session.rate_limiter,
payment_rate_manager,
wallet,
session.wallet,
timeout)
mirror = HTTPBlobDownloader(blob_manager, [blob_hash], download_mirrors or [], sd_hashes=[blob_hash], retry=False)
mirror.start()
sd_blob = yield downloader.download()
mirror.stop()
sd_reader = BlobStreamDescriptorReader(sd_blob)
sd_info = yield sd_reader.get_info()
try:
validate_descriptor(sd_info)
except InvalidStreamDescriptorError as err:
yield blob_manager.delete_blobs([blob_hash])
yield session.blob_manager.delete_blobs([blob_hash])
raise err
raw_sd = yield sd_reader._get_raw_data()
yield f2d(blob_manager.storage.add_known_blob(blob_hash, len(raw_sd)))
yield save_sd_info(blob_manager, sd_blob.blob_hash, sd_info)
yield session.blob_manager.storage.add_known_blob(blob_hash, len(raw_sd))
yield save_sd_info(session.blob_manager, sd_blob.blob_hash, sd_info)
defer.returnValue(sd_blob)
File diff suppressed because it is too large. Load diff
File renamed without changes.
@@ -5,12 +5,14 @@ from decimal import Decimal
from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.internet.error import ConnectionAborted
from zope.interface import implements
from lbrynet.p2p.Error import ConnectionClosedBeforeResponseError
from lbrynet.p2p.Error import InvalidResponseError, RequestCanceledError, NoResponseError
from lbrynet.p2p.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
from lbrynet.p2p.client.ClientRequest import ClientRequest, ClientBlobRequest
from lbrynet.p2p.Offer import Offer
from lbrynet.core.Error import ConnectionClosedBeforeResponseError
from lbrynet.core.Error import InvalidResponseError, RequestCanceledError, NoResponseError
from lbrynet.core.Error import PriceDisagreementError, DownloadCanceledError, InsufficientFundsError
from lbrynet.core.client.ClientRequest import ClientRequest, ClientBlobRequest
from lbrynet.interfaces import IRequestCreator
from lbrynet.core.Offer import Offer
log = logging.getLogger(__name__)
@@ -27,7 +29,7 @@ def get_points(num_bytes, rate):
def cache(fn):
"""Caches the function call for each instance"""
attr = f'__{fn.__name__}_value'
attr = '__{}_value'.format(fn.__name__)
def helper(self):
if not hasattr(self, attr):
@@ -37,8 +39,8 @@ def cache(fn):
return helper
class BlobRequester:
#implements(IRequestCreator)
class BlobRequester(object):
implements(IRequestCreator)
def __init__(self, blob_manager, peer_finder, payment_rate_manager, wallet, download_manager):
self.blob_manager = blob_manager
@@ -161,7 +163,7 @@ class BlobRequester:
return True
def _get_bad_peers(self):
return [p for p in self._peers.keys() if not self._should_send_request_to(p)]
return [p for p in self._peers.iterkeys() if not self._should_send_request_to(p)]
def _hash_available(self, blob_hash):
for peer in self._available_blobs:
@@ -193,7 +195,7 @@ class BlobRequester:
self._peers[peer] += amount
class RequestHelper:
class RequestHelper(object):
def __init__(self, requestor, peer, protocol, payment_rate_manager):
self.requestor = requestor
self.peer = peer
@@ -225,7 +227,7 @@ class RequestHelper:
def _request_failed(self, reason, request_type):
if reason.check(DownloadCanceledError, RequestCanceledError, ConnectionAborted,
ConnectionClosedBeforeResponseError, ValueError):
ConnectionClosedBeforeResponseError):
return
if reason.check(NoResponseError):
self.requestor._incompatible_peers.append(self.peer)
@@ -236,6 +238,8 @@ class RequestHelper:
self.peer.update_score(-10.0)
else:
self.peer.update_score(-2.0)
if reason.check(ConnectionClosedBeforeResponseError):
return
return reason
def get_rate(self):
@@ -285,8 +289,7 @@ def _handle_incoming_blob(response_dict, peer, request):
def _handle_download_error(err, peer, blob_to_download):
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError,
ConnectionClosedBeforeResponseError):
if not err.check(DownloadCanceledError, PriceDisagreementError, RequestCanceledError):
log.warning("An error occurred while downloading %s from %s. Error: %s",
blob_to_download.blob_hash, str(peer), err.getTraceback())
if err.check(PriceDisagreementError):
@@ -352,10 +355,6 @@ class AvailabilityRequest(RequestHelper):
log.debug("Received a response to the availability request")
# save available blobs
blob_hashes = response_dict['available_blobs']
if not blob_hashes:
# should not send any more requests as it doesn't have any blob we need
self.update_local_score(-10.0)
return True
for blob_hash in blob_hashes:
if blob_hash in request.request_dict['requested_blobs']:
self.process_available_blob_hash(blob_hash, request)
@@ -427,7 +426,7 @@ class PriceRequest(RequestHelper):
class DownloadRequest(RequestHelper):
"""Choose a blob and download it from a peer and also pay the peer for the data."""
def __init__(self, requester, peer, protocol, payment_rate_manager, wallet, head_blob_hash):
super().__init__(requester, peer, protocol, payment_rate_manager)
RequestHelper.__init__(self, requester, peer, protocol, payment_rate_manager)
self.wallet = wallet
self.head_blob_hash = head_blob_hash
@@ -576,7 +575,7 @@ class DownloadRequest(RequestHelper):
return reason
class BlobDownloadDetails:
class BlobDownloadDetails(object):
"""Contains the information needed to make a ClientBlobRequest from an open blob"""
def __init__(self, blob, deferred, write_func, cancel_func, peer):
self.blob = blob
@@ -5,11 +5,13 @@ from twisted.internet import error, defer
from twisted.internet.protocol import Protocol, ClientFactory
from twisted.protocols.policies import TimeoutMixin
from twisted.python import failure
from lbrynet import utils
from lbrynet.conf import MAX_RESPONSE_INFO_SIZE
from lbrynet.p2p.Error import ConnectionClosedBeforeResponseError, NoResponseError
from lbrynet.p2p.Error import DownloadCanceledError, MisbehavingPeerError
from lbrynet.p2p.Error import RequestCanceledError
from lbrynet import conf
from lbrynet.core import utils
from lbrynet.core.Error import ConnectionClosedBeforeResponseError, NoResponseError
from lbrynet.core.Error import DownloadCanceledError, MisbehavingPeerError
from lbrynet.core.Error import RequestCanceledError
from lbrynet.interfaces import IRequestSender, IRateLimited
from zope.interface import implements
log = logging.getLogger(__name__)
@@ -22,7 +24,7 @@ def encode_decimal(obj):
class ClientProtocol(Protocol, TimeoutMixin):
#implements(IRequestSender, IRateLimited)
implements(IRequestSender, IRateLimited)
######### Protocol #########
PROTOCOL_TIMEOUT = 30
@@ -32,7 +34,7 @@ class ClientProtocol(Protocol, TimeoutMixin):
self._rate_limiter = self.factory.rate_limiter
self.peer = self.factory.peer
self._response_deferreds = {}
self._response_buff = b''
self._response_buff = ''
self._downloading_blob = False
self._blob_download_request = None
self._next_request = {}
@@ -53,13 +55,13 @@ class ClientProtocol(Protocol, TimeoutMixin):
self._blob_download_request.write(data)
else:
self._response_buff += data
if len(self._response_buff) > MAX_RESPONSE_INFO_SIZE:
if len(self._response_buff) > conf.settings['MAX_RESPONSE_INFO_SIZE']:
log.warning("Response is too large from %s. Size %s",
self.peer, len(self._response_buff))
self.transport.loseConnection()
response, extra_data = self._get_valid_response(self._response_buff)
if response is not None:
self._response_buff = b''
self._response_buff = ''
self._handle_response(response)
if self._downloading_blob is True and len(extra_data) != 0:
self._blob_download_request.write(extra_data)
@@ -69,17 +71,17 @@ class ClientProtocol(Protocol, TimeoutMixin):
self.peer.report_down()
self.transport.abortConnection()
def connectionLost(self, reason=None):
def connectionLost(self, reason):
log.debug("Connection lost to %s: %s", self.peer, reason)
self.setTimeout(None)
self.connection_closed = True
if reason is None or reason.check(error.ConnectionDone):
if reason.check(error.ConnectionDone):
err = failure.Failure(ConnectionClosedBeforeResponseError())
else:
err = reason
for key, d in self._response_deferreds.items():
del self._response_deferreds[key]
d.errback(err)
self._response_deferreds.clear()
if self._blob_download_request is not None:
self._blob_download_request.cancel(err)
self.factory.connection_was_made_deferred.callback(True)
@@ -103,19 +105,19 @@ class ClientProtocol(Protocol, TimeoutMixin):
self._handle_response_error)
return d
else:
return defer.fail(ValueError("There is already a blob download request active"))
raise ValueError("There is already a blob download request active")
def cancel_requests(self):
self.connection_closing = True
ds = []
err = RequestCanceledError()
for key, d in list(self._response_deferreds.items()):
for key, d in self._response_deferreds.items():
del self._response_deferreds[key]
d.errback(err)
ds.append(d)
if self._blob_download_request is not None:
ds.append(self._blob_download_request.finished_deferred)
self._blob_download_request.cancel(err)
ds.append(self._blob_download_request.finished_deferred)
self._blob_download_request = None
self._downloading_blob = False
return defer.DeferredList(ds)
@@ -123,8 +125,9 @@ class ClientProtocol(Protocol, TimeoutMixin):
######### Internal request handling #########
def _handle_request_error(self, err):
log.error("An unexpected error occurred creating or sending a request to %s. %s: %s",
self.peer, err.type, err)
log.error(
"An unexpected error occurred creating or sending a request to %s. Error message: %s",
self.peer, err.getTraceback())
self.transport.loseConnection()
def _ask_for_request(self):
@@ -149,7 +152,7 @@ class ClientProtocol(Protocol, TimeoutMixin):
self.setTimeout(self.PROTOCOL_TIMEOUT)
# TODO: compare this message to the last one. If they're the same,
# TODO: incrementally delay this message.
m = json.dumps(request_msg, default=encode_decimal).encode()
m = json.dumps(request_msg, default=encode_decimal)
self.transport.write(m)
def _get_valid_response(self, response_msg):
@@ -157,7 +160,7 @@ class ClientProtocol(Protocol, TimeoutMixin):
response = None
curr_pos = 0
while 1:
next_close_paren = response_msg.find(b'}', curr_pos)
next_close_paren = response_msg.find('}', curr_pos)
if next_close_paren != -1:
curr_pos = next_close_paren + 1
try:
@@ -173,8 +176,7 @@ class ClientProtocol(Protocol, TimeoutMixin):
def _handle_response_error(self, err):
# If an error gets to this point, log it and kill the connection.
if err.check(DownloadCanceledError, RequestCanceledError, error.ConnectionAborted,
ConnectionClosedBeforeResponseError):
if err.check(DownloadCanceledError, RequestCanceledError, error.ConnectionAborted):
# TODO: (wish-list) it seems silly to close the connection over this, and it shouldn't
# TODO: always be this way. it's done this way now because the client has no other way
# TODO: of telling the server it wants the download to stop. It would be great if the
@@ -182,15 +184,14 @@ class ClientProtocol(Protocol, TimeoutMixin):
log.info("Closing the connection to %s because the download of blob %s was canceled",
self.peer, self._blob_download_request.blob)
result = None
elif err.check(MisbehavingPeerError):
elif not err.check(MisbehavingPeerError, ConnectionClosedBeforeResponseError):
log.warning("The connection to %s is closing due to: %s", self.peer, err)
result = err
else:
log.error("The connection to %s is closing due to an unexpected error: %s",
self.peer, err)
result = err
self._blob_download_request = None
self._downloading_blob = False
self.transport.loseConnection()
return result
@@ -1,7 +1,7 @@
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
class ClientRequest:
class ClientRequest(object):
def __init__(self, request_dict, response_identifier=None):
self.request_dict = request_dict
self.response_identifier = response_identifier
@@ -9,7 +9,7 @@ class ClientRequest:
class ClientPaidRequest(ClientRequest):
def __init__(self, request_dict, response_identifier, max_pay_units):
super().__init__(request_dict, response_identifier)
ClientRequest.__init__(self, request_dict, response_identifier)
self.max_pay_units = max_pay_units
@@ -20,7 +20,7 @@ class ClientBlobRequest(ClientPaidRequest):
max_pay_units = MAX_BLOB_SIZE
else:
max_pay_units = blob.length
super().__init__(request_dict, response_identifier, max_pay_units)
ClientPaidRequest.__init__(self, request_dict, response_identifier, max_pay_units)
self.write = write_func
self.finished_deferred = finished_deferred
self.cancel = cancel_func
@@ -1,31 +1,33 @@
import random
import logging
from twisted.internet import defer, reactor
from lbrynet import utils
from lbrynet.conf import Config
from lbrynet.p2p.client.ClientProtocol import ClientProtocolFactory
from lbrynet.p2p.Error import InsufficientFundsError
from zope.interface import implements
from lbrynet import interfaces
from lbrynet import conf
from lbrynet.core.client.ClientProtocol import ClientProtocolFactory
from lbrynet.core.Error import InsufficientFundsError
from lbrynet.core import utils
log = logging.getLogger(__name__)
class PeerConnectionHandler:
class PeerConnectionHandler(object):
def __init__(self, request_creators, factory):
self.request_creators = request_creators
self.factory = factory
self.connection = None
class ConnectionManager:
#implements(interfaces.IConnectionManager)
class ConnectionManager(object):
implements(interfaces.IConnectionManager)
MANAGE_CALL_INTERVAL_SEC = 5
TCP_CONNECT_TIMEOUT = 15
def __init__(self, downloader, rate_limiter, primary_request_creators, secondary_request_creators):
def __init__(self, downloader, rate_limiter,
primary_request_creators, secondary_request_creators):
self.conf: Config = downloader.conf
self.seek_head_blob_first = self.conf.seek_head_blob_first
self.max_connections_per_stream = self.conf.max_connections_per_stream
self.seek_head_blob_first = conf.settings['seek_head_blob_first']
self.max_connections_per_stream = conf.settings['max_connections_per_stream']
self.downloader = downloader
self.rate_limiter = rate_limiter
@@ -96,8 +98,7 @@ class ConnectionManager:
d.addBoth(lambda _: disconnect_peer(p))
return d
# fixme: stop modifying dict during iteration
closing_deferreds = [close_connection(peer) for peer in list(self._peer_connections)]
closing_deferreds = [close_connection(peer) for peer in self._peer_connections.keys()]
return defer.DeferredList(closing_deferreds)
@defer.inlineCallbacks
@@ -225,3 +226,5 @@ class ConnectionManager:
del self._connections_closing[peer]
d.callback(True)
return connection_was_made
+69
View File
@@ -0,0 +1,69 @@
import binascii
import logging
from zope.interface import implements
from twisted.internet import defer, reactor
from lbrynet.interfaces import IPeerFinder
from lbrynet.core.utils import short_hash
log = logging.getLogger(__name__)
class DHTPeerFinder(object):
"""This class finds peers which have announced to the DHT that they have certain blobs"""
implements(IPeerFinder)
def __init__(self, dht_node, peer_manager):
"""
dht_node - an instance of dht.Node class
peer_manager - an instance of PeerManager class
"""
self.dht_node = dht_node
self.peer_manager = peer_manager
self.peers = []
def stop(self):
pass
@defer.inlineCallbacks
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=False):
"""
Find peers for blob in the DHT
blob_hash (str): blob hash to look for
timeout (int): seconds to timeout after
filter_self (bool): if True, and if a peer for a blob is itself, filter it
from the result
Returns:
list of peers for the blob
"""
def _trigger_timeout():
if not finished_deferred.called:
log.debug("Peer search for %s timed out", short_hash(blob_hash))
finished_deferred.cancel()
bin_hash = binascii.unhexlify(blob_hash)
finished_deferred = self.dht_node.getPeersForBlob(bin_hash)
if timeout is not None:
reactor.callLater(timeout, _trigger_timeout)
try:
peer_list = yield finished_deferred
except defer.CancelledError:
peer_list = []
peers = set(peer_list)
good_peers = []
for host, port in peers:
if filter_self and (host, port) == (self.dht_node.externalIP, self.dht_node.peerPort):
continue
peer = self.peer_manager.get_peer(host, port)
if peer.is_available() is True:
good_peers.append(peer)
defer.returnValue(good_peers)
def get_most_popular_hashes(self, num_to_return):
return self.dht_node.get_most_popular_hashes(num_to_return)
@@ -1,12 +1,14 @@
import logging
from twisted.internet import defer
from zope.interface import implements
from lbrynet import interfaces
log = logging.getLogger(__name__)
class DownloadManager:
#implements(interfaces.IDownloadManager)
class DownloadManager(object):
implements(interfaces.IDownloadManager)
def __init__(self, blob_manager):
self.blob_manager = blob_manager
@@ -30,7 +32,7 @@ class DownloadManager:
def resume_downloading(self):
yield self.connection_manager.start()
yield self.progress_manager.start()
return True
defer.returnValue(True)
@defer.inlineCallbacks
def stop_downloading(self):
@@ -39,14 +41,32 @@ class DownloadManager:
defer.returnValue(True)
def add_blobs_to_download(self, blob_infos):
log.debug("Adding %s blobs to blobs", len(blob_infos))
def add_blob_to_list(blob, blob_num):
self.blobs[blob_num] = blob
log.debug(
"Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_num)
def error_during_add(err):
log.warning(
"An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
return err
ds = []
for blob_info in blob_infos:
if not blob_info.blob_num in self.blobs:
self.blob_infos[blob_info.blob_num] = blob_info
log.debug("Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
blob = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
self.blobs[blob_info.blob_num] = blob
log.debug("Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_info.blob_num)
log.debug(
"Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
d = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
d.addCallback(add_blob_to_list, blob_info.blob_num)
d.addErrback(error_during_add)
ds.append(d)
dl = defer.DeferredList(ds)
return dl
def stream_position(self):
return self.progress_manager.stream_position()
@@ -61,14 +81,14 @@ class DownloadManager:
return self.blob_handler.handle_blob(self.blobs[blob_num], self.blob_infos[blob_num])
def calculate_total_bytes(self):
return sum([bi.length for bi in self.blob_infos.values()])
return sum([bi.length for bi in self.blob_infos.itervalues()])
def calculate_bytes_left_to_output(self):
if not self.blobs:
return self.calculate_total_bytes()
else:
to_be_outputted = [
b for n, b in self.blobs.items()
b for n, b in self.blobs.iteritems()
if n >= self.progress_manager.last_blob_outputted
]
return sum([b.length for b in to_be_outputted if b.length is not None])
@@ -1,12 +1,12 @@
import logging
from lbrynet.conf import Config
from lbrynet.p2p.BlobInfo import BlobInfo
from lbrynet.p2p.client.BlobRequester import BlobRequester
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
from lbrynet.p2p.client.DownloadManager import DownloadManager
from lbrynet.p2p.Error import InvalidBlobHashError, DownloadSDTimeout
from lbrynet.blob.blob_file import is_valid_blobhash
from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
from zope.interface import implements
from lbrynet import interfaces
from lbrynet.core.BlobInfo import BlobInfo
from lbrynet.core.client.BlobRequester import BlobRequester
from lbrynet.core.client.ConnectionManager import ConnectionManager
from lbrynet.core.client.DownloadManager import DownloadManager
from lbrynet.core.Error import InvalidBlobHashError, DownloadSDTimeout
from lbrynet.core.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call
from twisted.python.failure import Failure
from twisted.internet import defer
from twisted.internet.task import LoopingCall
@@ -14,8 +14,8 @@ from twisted.internet.task import LoopingCall
log = logging.getLogger(__name__)
class SingleBlobMetadataHandler:
#implements(interfaces.IMetadataHandler)
class SingleBlobMetadataHandler(object):
implements(interfaces.IMetadataHandler)
def __init__(self, blob_hash, download_manager):
self.blob_hash = blob_hash
@@ -31,7 +31,7 @@ class SingleBlobMetadataHandler:
return 0
class SingleProgressManager:
class SingleProgressManager(object):
def __init__(self, download_manager, finished_callback, timeout_callback, timeout):
self.finished_callback = finished_callback
self.timeout_callback = timeout_callback
@@ -71,10 +71,10 @@ class SingleProgressManager:
def needed_blobs(self):
blobs = self.download_manager.blobs
assert len(blobs) == 1
return [b for b in blobs.values() if not b.get_is_verified()]
return [b for b in blobs.itervalues() if not b.get_is_verified()]
class DummyBlobHandler:
class DummyBlobHandler(object):
def __init__(self):
pass
@@ -82,11 +82,10 @@ class DummyBlobHandler:
pass
class StandaloneBlobDownloader:
def __init__(self, conf: Config, blob_hash, blob_manager, peer_finder,
class StandaloneBlobDownloader(object):
def __init__(self, blob_hash, blob_manager, peer_finder,
rate_limiter, payment_rate_manager, wallet,
timeout=None):
self.conf = conf
self.blob_hash = blob_hash
self.blob_manager = blob_manager
self.peer_finder = peer_finder
@@ -1,16 +1,17 @@
import logging
from twisted.internet import defer, task
from lbrynet.interfaces import IProgressManager
from twisted.internet import defer
from zope.interface import implements
log = logging.getLogger(__name__)
class FullStreamProgressManager:
def __init__(self, finished_callback, blob_manager, download_manager,
delete_blob_after_finished: bool = False, reactor: task.Clock = None):
if not reactor:
from twisted.internet import reactor
self.reactor = reactor
class StreamProgressManager(object):
implements(IProgressManager)
def __init__(self, finished_callback, blob_manager,
download_manager, delete_blob_after_finished=False):
self.finished_callback = finished_callback
self.blob_manager = blob_manager
self.delete_blob_after_finished = delete_blob_after_finished
@@ -20,11 +21,15 @@ class FullStreamProgressManager:
self.stopped = True
self._next_try_to_output_call = None
self.outputting_d = None
self.wrote_first_data = defer.Deferred()
######### IProgressManager #########
def start(self):
from twisted.internet import reactor
self.stopped = False
self._next_try_to_output_call = self.reactor.callLater(0, self._try_to_output)
self._next_try_to_output_call = reactor.callLater(0, self._try_to_output)
return defer.succeed(True)
def stop(self):
@@ -34,43 +39,32 @@ class FullStreamProgressManager:
self._next_try_to_output_call = None
return self._stop_outputting()
# def blob_downloaded(self, blob, blob_num):
# if self.outputting_d is None:
# self._output_loop()
def blob_downloaded(self, blob, blob_num):
if self.outputting_d is None:
self._output_loop()
def stream_position(self):
blobs = self.download_manager.blobs
if not blobs:
return 0
else:
for i in range(max(blobs.keys())):
if self._done(i, blobs):
return i
return max(blobs.keys()) + 1
def needed_blobs(self):
blobs = self.download_manager.blobs
return [
b for n, b in blobs.items()
if not b.get_is_verified() and not n in self.provided_blob_nums
]
######### internal #########
def _finished_outputting(self):
self.finished_callback(True)
def _try_to_output(self):
self._next_try_to_output_call = self.reactor.callLater(1, self._try_to_output)
from twisted.internet import reactor
self._next_try_to_output_call = reactor.callLater(1, self._try_to_output)
if self.outputting_d is None:
self._output_loop()
def _output_loop(self):
pass
def _stop_outputting(self):
if self.outputting_d is not None:
return self.outputting_d
return defer.succeed(None)
def _finished_with_blob(self, blob_num: int) -> None:
if blob_num == 0 and not self.wrote_first_data.called:
self.wrote_first_data.callback(True)
def _finished_with_blob(self, blob_num):
log.debug("In _finished_with_blob, blob_num = %s", str(blob_num))
if self.delete_blob_after_finished is True:
log.debug("delete_blob_after_finished is True")
@@ -84,17 +78,49 @@ class FullStreamProgressManager:
else:
log.debug("delete_blob_after_finished is False")
def _done(self, i: int, blobs: list) -> bool:
class FullStreamProgressManager(StreamProgressManager):
def __init__(self, finished_callback, blob_manager,
download_manager, delete_blob_after_finished=False):
StreamProgressManager.__init__(self, finished_callback, blob_manager, download_manager,
delete_blob_after_finished)
self.outputting_d = None
######### IProgressManager #########
def _done(self, i, blobs):
"""Return true if `i` is a blob number we don't have"""
return (
i not in blobs or
(
not blobs[i].get_is_verified() and
i not in self.provided_blob_nums
)
i not in blobs or
(
not blobs[i].get_is_verified() and
i not in self.provided_blob_nums
)
)
def stream_position(self):
blobs = self.download_manager.blobs
if not blobs:
return 0
else:
for i in xrange(max(blobs.iterkeys())):
if self._done(i, blobs):
return i
return max(blobs.iterkeys()) + 1
def needed_blobs(self):
blobs = self.download_manager.blobs
return [
b for n, b in blobs.iteritems()
if not b.get_is_verified() and not n in self.provided_blob_nums
]
######### internal #########
def _output_loop(self):
from twisted.internet import reactor
if self.stopped:
if self.outputting_d is not None:
self.outputting_d.callback(True)
@@ -115,7 +141,7 @@ class FullStreamProgressManager:
self.outputting_d.callback(True)
self.outputting_d = None
else:
self.reactor.callLater(0, self._output_loop)
reactor.callLater(0, self._output_loop)
current_blob_num = self.last_blob_outputted + 1
File renamed without changes.
File renamed without changes.
+17
View File
@@ -0,0 +1,17 @@
import os
from contextlib import contextmanager
@contextmanager
def get_read_handle(path):
"""
Get os independent read handle for a file
"""
if os.name == "nt":
file_mode = 'rb'
else:
file_mode = 'r'
read_handle = open(path, file_mode)
yield read_handle
read_handle.close()
+383
View File
@@ -0,0 +1,383 @@
import inspect
import json
import logging
import logging.handlers
import os
import sys
import traceback
from txrequests import Session
from requests.exceptions import ConnectionError
from twisted.internet import defer
import twisted.python.log
from lbrynet import __version__ as lbrynet_version, build_type, conf
from lbrynet.core import utils
####
# This code is copied from logging/__init__.py in the python source code
####
#
# _srcfile is used when walking the stack to check when we've got the first
# caller stack frame.
#
if hasattr(sys, 'frozen'): # support for py2exe
_srcfile = "logging%s__init__%s" % (os.sep, __file__[-4:])
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
_srcfile = __file__[:-4] + '.py'
else:
_srcfile = __file__
_srcfile = os.path.normcase(_srcfile)
#####
TRACE = 5
class HTTPSHandler(logging.Handler):
def __init__(self, url, fqdn=False, localname=None, facility=None, session=None):
logging.Handler.__init__(self)
self.url = url
self.fqdn = fqdn
self.localname = localname
self.facility = facility
self.session = session if session is not None else Session()
def get_full_message(self, record):
if record.exc_info:
return '\n'.join(traceback.format_exception(*record.exc_info))
else:
return record.getMessage()
@defer.inlineCallbacks
def _emit(self, record):
payload = self.format(record)
try:
yield self.session.post(self.url, data=payload)
except ConnectionError:
pass
def emit(self, record):
return self._emit(record)
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
def remove_handlers(log, handler_name):
for handler in log.handlers:
if handler.name == handler_name:
log.removeHandler(handler)
def _log_decorator(fn):
"""Configure a logging handler.
`fn` is a function that returns a logging handler. The returned
handler has its log-level set and is attached to the specified
logger or the root logger.
"""
def helper(*args, **kwargs):
log = kwargs.pop('log', logging.getLogger())
level = kwargs.pop('level', logging.INFO)
if not isinstance(level, int):
# despite the name, getLevelName returns
# the numeric level when passed a text level
level = logging.getLevelName(level)
handler = fn(*args, **kwargs)
configure_handler(handler, log, level)
return handler
return helper
def configure_handler(handler, log, level):
if handler.name:
remove_handlers(log, handler.name)
handler.setLevel(level)
log.addHandler(handler)
# need to reduce the logger's level down to the
# handler's level or else the handler won't
# get those messages
if log.level > level:
log.setLevel(level)
return handler
def disable_third_party_loggers():
logging.getLogger('requests').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('BitcoinRPC').setLevel(logging.INFO)
logging.getLogger('lbryum').setLevel(logging.WARNING)
logging.getLogger('twisted').setLevel(logging.CRITICAL)
@_log_decorator
def configure_console(**kwargs):
"""Convenience function to configure a log-handler that outputs to stdout"""
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(DEFAULT_FORMATTER)
handler.name = 'console'
return handler
@_log_decorator
def configure_file_handler(file_name, **kwargs):
"""Convenience function to configure a log-handler that writes to `file_name`"""
handler = logging.handlers.RotatingFileHandler(file_name, maxBytes=2097152, backupCount=5)
handler.setFormatter(DEFAULT_FORMATTER)
handler.name = 'file'
return handler
def get_loggly_url(token=None, version=None):
token = token or utils.deobfuscate(conf.settings['LOGGLY_TOKEN'])
version = version or lbrynet_version
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
def configure_loggly_handler():
if build_type.BUILD == 'dev':
return
level = logging.ERROR
handler = get_loggly_handler(level=level, installation_id=conf.settings.installation_id,
session_id=conf.settings.get_session_id())
log = logging.getLogger("lbrynet")
if handler.name:
remove_handlers(log, handler.name)
handler.setLevel(level)
log.addHandler(handler)
# need to reduce the logger's level down to the
# handler's level or else the handler won't
# get those messages
if log.level > level:
log.setLevel(level)
def get_loggly_handler(level, installation_id, session_id):
formatter = JsonFormatter(level=level, installation_id=installation_id, session_id=session_id)
handler = HTTPSHandler(get_loggly_url())
handler.setFormatter(formatter)
handler.name = 'loggly'
return handler
class JsonFormatter(logging.Formatter):
"""Format log records using json serialization"""
def __init__(self, **kwargs):
self.attributes = kwargs
def format(self, record):
data = {
'loggerName': record.name,
'asciTime': self.formatTime(record),
'fileName': record.filename,
'functionName': record.funcName,
'levelNo': record.levelno,
'lineNo': record.lineno,
'levelName': record.levelname,
'message': record.getMessage(),
}
data.update(self.attributes)
if record.exc_info:
data['exc_info'] = self.formatException(record.exc_info)
return json.dumps(data)
####
# This code is copied from logging/__init__.py in the python source code
####
def findCaller(srcfile=None):
"""Returns the filename, line number and function name of the caller"""
srcfile = srcfile or _srcfile
f = inspect.currentframe()
# On some versions of IronPython, currentframe() returns None if
# IronPython isn't run with -X:Frames.
if f is not None:
f = f.f_back
rv = "(unknown file)", 0, "(unknown function)"
while hasattr(f, "f_code"):
co = f.f_code
filename = os.path.normcase(co.co_filename)
# ignore any function calls that are in this file
if filename == srcfile:
f = f.f_back
continue
rv = (filename, f.f_lineno, co.co_name)
break
return rv
###
def failure(failure, log, msg, *args):
"""Log a failure message from a deferred.
Args:
failure: twisted.python.failure.Failure
log: a python logger instance
msg: the message to log. Can use normal logging string interpolation.
the last argument will be set to the error message from the failure.
args: values to substitute into `msg`
"""
args += (failure.getErrorMessage(),)
exc_info = (failure.type, failure.value, failure.getTracebackObject())
log.error(msg, *args, exc_info=exc_info)
def convert_verbose(verbose):
"""Convert the results of the --verbose flag into a list of logger names
if --verbose is not provided, args.verbose will be None and logging
should be at the info level.
if --verbose is provided, but not followed by any arguments, then
args.verbose = [] and debug logging should be enabled for all of lbrynet
along with info logging on lbryum.
if --verbose is provided and followed by arguments, those arguments
will be in a list
"""
if verbose is None:
return []
if verbose == []:
return ['lbrynet', 'lbryum']
return verbose
def configure_logging(file_name, console, verbose=None):
"""Apply the default logging configuration.
Enables two log-handlers at the INFO level: a file logger and a loggly logger.
Optionally turns on a console logger that defaults to the INFO level, with
specified loggers being set to the DEBUG level.
Args:
file_name: the file to which logs should be saved
console: If true, enable a console logger
verbose: a list of loggers to set to debug level.
See `convert_verbose` for more details.
"""
verbose = convert_verbose(verbose)
configure_twisted()
configure_file_handler(file_name)
disable_third_party_loggers()
if console:
# if there are some loggers at the debug level, we need
# to enable the console to allow debug. Otherwise, only
# allow info.
level = 'DEBUG' if verbose else 'INFO'
handler = configure_console(level=level)
if 'lbryum' in verbose:
# TODO: this enables lbryum logging on the other handlers
# too which isn't consistent with how verbose logging
# happens with other loggers. Should change the configuration
# so that its only logging at the INFO level for the console.
logging.getLogger('lbryum').setLevel(logging.INFO)
verbose.remove('lbryum')
if verbose:
handler.addFilter(LoggerNameFilter(verbose))
def configure_twisted():
"""Setup twisted logging to output events to the python stdlib logger"""
# I tried using the new logging api
# https://twistedmatrix.com/documents/current/core/howto/logger.html#compatibility-with-standard-library-logging
# and it simply didn't work
observer = twisted.python.log.PythonLoggingObserver()
observer.start()
class LoggerNameFilter(object):
"""Filter a log record based on its name.
Allows all info level and higher records to pass thru.
Debug records pass if the log record name (or a parent) match
the input list of logger names.
"""
def __init__(self, logger_names):
self.logger_names = logger_names
def filter(self, record):
if record.levelno >= logging.INFO:
return True
name = record.name
while name:
if name in self.logger_names:
return True
name = get_parent(name)
return False
def get_parent(logger_name):
names = logger_name.split('.')
if len(names) == 1:
return ''
names = names[:-1]
return '.'.join(names)
class Logger(logging.Logger):
"""A logger that has an extra `fail` method useful for handling twisted failures."""
def fail(self, callback=None, *args, **kwargs):
"""Returns a function to log a failure from an errback.
The returned function appends the error message and extracts
the traceback from `err`.
Example usage:
d.addErrback(log.fail(), 'This is an error message')
Although odd, making the method call is necessary to extract
out useful filename and line number information; otherwise the
reported values are from inside twisted's deferred handling
code.
Args:
callback: callable to call after making the log. The first argument
will be the `err` from the deferred
args: extra arguments to pass into `callback`
Returns: a function that takes the following arguments:
err: twisted.python.failure.Failure
msg: the message to log, using normal logging string iterpolation.
msg_args: the values to subtitute into `msg`
msg_kwargs: set `level` to change from the default ERROR severity. Other
keywoards are treated as normal log kwargs.
"""
fn, lno, func = findCaller()
def _fail(err, msg, *msg_args, **msg_kwargs):
level = msg_kwargs.pop('level', logging.ERROR)
msg += ": %s"
msg_args += (err.getErrorMessage(),)
exc_info = (err.type, err.value, err.getTracebackObject())
record = self.makeRecord(
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
self.handle(record)
if callback:
try:
return callback(err, *args, **kwargs)
except Exception:
# log.fail is almost always called within an
# errback. If callback fails and we didn't catch
# the exception we would need to attach a second
# errback to deal with that, which we will almost
# never do and then we end up with an unhandled
# error that will get swallowed by twisted
self.exception('Failed to run callback')
return _fail
def trace(self, msg, *args, **kwargs):
if self.isEnabledFor(TRACE):
self._log(TRACE, msg, args, **kwargs)
logging.setLoggerClass(Logger)
logging.addLevelName(TRACE, 'TRACE')
@@ -1,9 +1,9 @@
class LoopingCallManager:
class LoopingCallManager(object):
def __init__(self, calls=None):
self.calls = calls or {}
def register_looping_call(self, name, call):
assert name not in self.calls, f'{name} is already registered'
assert name not in self.calls, '{} is already registered'.format(name)
self.calls[name] = call
def start(self, name, *args):
@@ -15,6 +15,6 @@ class LoopingCallManager:
self.calls[name].stop()
def shutdown(self):
for lcall in self.calls.values():
for lcall in self.calls.itervalues():
if lcall.running:
lcall.stop()
@@ -1,11 +1,14 @@
import logging
from twisted.internet import defer
from zope.interface import implements
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler
log = logging.getLogger(__name__)
class BlobAvailabilityHandlerFactory:
# implements(IQueryHandlerFactory)
class BlobAvailabilityHandlerFactory(object):
implements(IQueryHandlerFactory)
def __init__(self, blob_manager):
self.blob_manager = blob_manager
@@ -23,8 +26,8 @@ class BlobAvailabilityHandlerFactory:
return "Blob Availability - blobs that are available to be uploaded"
class BlobAvailabilityHandler:
#implements(IQueryHandler)
class BlobAvailabilityHandler(object):
implements(IQueryHandler)
def __init__(self, blob_manager):
self.blob_manager = blob_manager
@@ -38,12 +41,19 @@ class BlobAvailabilityHandler:
def handle_queries(self, queries):
if self.query_identifiers[0] in queries:
log.info("Received the client's list of requested blobs")
available_blobs = self._get_available_blobs(queries[self.query_identifiers[0]])
log.debug("available blobs: %s", str(available_blobs))
return {'available_blobs': available_blobs}
return {}
d = self._get_available_blobs(queries[self.query_identifiers[0]])
def set_field(available_blobs):
log.debug("available blobs: %s", str(available_blobs))
return {'available_blobs': available_blobs}
d.addCallback(set_field)
return d
return defer.succeed({})
######### internal #########
def _get_available_blobs(self, requested_blobs):
return self.blob_manager.completed_blobs(requested_blobs)
d = self.blob_manager.completed_blobs(requested_blobs)
return d
@@ -3,15 +3,17 @@ import logging
from twisted.internet import defer
from twisted.protocols.basic import FileSender
from twisted.python.failure import Failure
from zope.interface import implements
from lbrynet.extras.daemon import analytics
from lbrynet.p2p.Offer import Offer
from lbrynet import analytics
from lbrynet.core.Offer import Offer
from lbrynet.interfaces import IQueryHandlerFactory, IQueryHandler, IBlobSender
log = logging.getLogger(__name__)
class BlobRequestHandlerFactory:
#implements(IQueryHandlerFactory)
class BlobRequestHandlerFactory(object):
implements(IQueryHandlerFactory)
def __init__(self, blob_manager, wallet, payment_rate_manager, analytics_manager):
self.blob_manager = blob_manager
@@ -33,8 +35,8 @@ class BlobRequestHandlerFactory:
return "Blob Uploader - uploads blobs"
class BlobRequestHandler:
#implements(IQueryHandler, IBlobSender)
class BlobRequestHandler(object):
implements(IQueryHandler, IBlobSender)
PAYMENT_RATE_QUERY = 'blob_data_payment_rate'
BLOB_QUERY = 'requested_blob'
AVAILABILITY_QUERY = 'requested_blobs'
@@ -61,21 +63,21 @@ class BlobRequestHandler:
request_handler.register_blob_sender(self)
def handle_queries(self, queries):
response = {}
response = defer.succeed({})
log.debug("Handle query: %s", str(queries))
if self.AVAILABILITY_QUERY in queries:
self._blobs_requested = queries[self.AVAILABILITY_QUERY]
self._reply_to_availability(response, self._blobs_requested)
response.addCallback(lambda r: self._reply_to_availability(r, self._blobs_requested))
if self.PAYMENT_RATE_QUERY in queries:
offered_rate = queries[self.PAYMENT_RATE_QUERY]
offer = Offer(offered_rate)
if offer.rate is None:
log.warning("Empty rate offer")
self._handle_payment_rate_query(offer, response)
response.addCallback(lambda r: self._handle_payment_rate_query(offer, r))
if self.BLOB_QUERY in queries:
incoming = queries[self.BLOB_QUERY]
self._reply_to_send_request(response, incoming)
response.addCallback(lambda r: self._reply_to_send_request(r, incoming))
return response
######### IBlobSender #########
@@ -95,10 +97,15 @@ class BlobRequestHandler:
######### internal #########
def _reply_to_availability(self, request, blobs):
available_blobs = self._get_available_blobs(blobs)
log.debug("available blobs: %s", str(available_blobs))
request.update({'available_blobs': available_blobs})
return request
d = self._get_available_blobs(blobs)
def set_available(available_blobs):
log.debug("available blobs: %s", str(available_blobs))
request.update({'available_blobs': available_blobs})
return request
d.addCallback(set_available)
return d
def _handle_payment_rate_query(self, offer, request):
blobs = self._blobs_requested
@@ -167,7 +174,8 @@ class BlobRequestHandler:
return d
def _get_available_blobs(self, requested_blobs):
return self.blob_manager.completed_blobs(requested_blobs)
d = self.blob_manager.completed_blobs(requested_blobs)
return d
def send_file(self, consumer):
+129
View File
@@ -0,0 +1,129 @@
import binascii
import collections
import logging
import time
from twisted.internet import defer
from lbrynet.core import utils
log = logging.getLogger(__name__)
class DHTHashAnnouncer(object):
ANNOUNCE_CHECK_INTERVAL = 60
CONCURRENT_ANNOUNCERS = 5
"""This class announces to the DHT that this peer has certain blobs"""
def __init__(self, dht_node, peer_port):
self.dht_node = dht_node
self.peer_port = peer_port
self.suppliers = []
self.next_manage_call = None
self.hash_queue = collections.deque()
self._concurrent_announcers = 0
def run_manage_loop(self):
if self.peer_port is not None:
self._announce_available_hashes()
self.next_manage_call = utils.call_later(self.ANNOUNCE_CHECK_INTERVAL, self.run_manage_loop)
def stop(self):
log.info("Stopping DHT hash announcer.")
if self.next_manage_call is not None:
self.next_manage_call.cancel()
self.next_manage_call = None
def add_supplier(self, supplier):
self.suppliers.append(supplier)
def immediate_announce(self, blob_hashes):
if self.peer_port is not None:
return self._announce_hashes(blob_hashes, immediate=True)
else:
return defer.succeed(False)
def hash_queue_size(self):
return len(self.hash_queue)
def _announce_available_hashes(self):
log.debug('Announcing available hashes')
ds = []
for supplier in self.suppliers:
d = supplier.hashes_to_announce()
d.addCallback(self._announce_hashes)
ds.append(d)
dl = defer.DeferredList(ds)
return dl
def _announce_hashes(self, hashes, immediate=False):
if not hashes:
return
log.debug('Announcing %s hashes', len(hashes))
# TODO: add a timeit decorator
start = time.time()
ds = []
for h in hashes:
announce_deferred = defer.Deferred()
ds.append(announce_deferred)
if immediate:
self.hash_queue.appendleft((h, announce_deferred))
else:
self.hash_queue.append((h, announce_deferred))
log.debug('There are now %s hashes remaining to be announced', self.hash_queue_size())
def announce():
if len(self.hash_queue):
h, announce_deferred = self.hash_queue.popleft()
log.debug('Announcing blob %s to dht', h)
d = self.dht_node.announceHaveBlob(binascii.unhexlify(h))
d.chainDeferred(announce_deferred)
d.addBoth(lambda _: utils.call_later(0, announce))
else:
self._concurrent_announcers -= 1
for i in range(self._concurrent_announcers, self.CONCURRENT_ANNOUNCERS):
self._concurrent_announcers += 1
announce()
d = defer.DeferredList(ds)
d.addCallback(lambda _: log.debug('Took %s seconds to announce %s hashes',
time.time() - start, len(hashes)))
return d
class DHTHashSupplier(object):
# 1 hour is the min time hash will be reannounced
MIN_HASH_REANNOUNCE_TIME = 60*60
# conservative assumption of the time it takes to announce
# a single hash
SINGLE_HASH_ANNOUNCE_DURATION = 5
"""Classes derived from this class give hashes to a hash announcer"""
def __init__(self, announcer):
if announcer is not None:
announcer.add_supplier(self)
self.hash_announcer = announcer
def hashes_to_announce(self):
pass
def get_next_announce_time(self, num_hashes_to_announce=1):
"""
Hash reannounce time is set to current time + MIN_HASH_REANNOUNCE_TIME,
unless we are announcing a lot of hashes at once which could cause the
the announce queue to pile up. To prevent pile up, reannounce
only after a conservative estimate of when it will finish
to announce all the hashes.
Args:
num_hashes_to_announce: number of hashes that will be added to the queue
Returns:
timestamp for next announce time
"""
queue_size = self.hash_announcer.hash_queue_size()+num_hashes_to_announce
reannounce = max(self.MIN_HASH_REANNOUNCE_TIME,
queue_size*self.SINGLE_HASH_ANNOUNCE_DURATION)
return time.time() + reannounce
@@ -1,8 +1,9 @@
import logging
from twisted.internet import error
from twisted.internet import interfaces, error
from twisted.internet.protocol import Protocol, ServerFactory
from twisted.python import failure
from lbrynet.p2p.server.ServerRequestHandler import ServerRequestHandler
from zope.interface import implements
from lbrynet.core.server.ServerRequestHandler import ServerRequestHandler
log = logging.getLogger(__name__)
@@ -23,7 +24,7 @@ class ServerProtocol(Protocol):
10) Pause/resume production when told by the rate limiter
"""
#implements(interfaces.IConsumer)
implements(interfaces.IConsumer)
#Protocol stuff
@@ -1,23 +1,25 @@
import json
import logging
from twisted.internet import defer
from twisted.internet import interfaces, defer
from zope.interface import implements
from lbrynet.interfaces import IRequestHandler
log = logging.getLogger(__name__)
class ServerRequestHandler:
class ServerRequestHandler(object):
"""This class handles requests from clients. It can upload blobs and
return request for information about more blobs that are
associated with streams.
"""
#implements(interfaces.IPushProducer, interfaces.IConsumer, IRequestHandler)
implements(interfaces.IPushProducer, interfaces.IConsumer, IRequestHandler)
def __init__(self, consumer):
self.consumer = consumer
self.production_paused = False
self.request_buff = b''
self.response_buff = b''
self.request_buff = ''
self.response_buff = ''
self.producer = None
self.request_received = False
self.CHUNK_SIZE = 2**14
@@ -54,7 +56,7 @@ class ServerRequestHandler:
return
chunk = self.response_buff[:self.CHUNK_SIZE]
self.response_buff = self.response_buff[self.CHUNK_SIZE:]
if chunk == b'':
if chunk == '':
return
log.trace("writing %s bytes to the client", len(chunk))
self.consumer.write(chunk)
@@ -99,7 +101,7 @@ class ServerRequestHandler:
self.request_buff = self.request_buff + data
msg = self.try_to_parse_request(self.request_buff)
if msg:
self.request_buff = b''
self.request_buff = ''
self._process_msg(msg)
else:
log.debug("Request buff not a valid json message")
@@ -132,7 +134,7 @@ class ServerRequestHandler:
self._produce_more()
def send_response(self, msg):
m = json.dumps(msg).encode()
m = json.dumps(msg)
log.debug("Sending a response of length %s", str(len(m)))
log.debug("Response: %s", str(m))
self.response_buff = self.response_buff + m
@@ -165,9 +167,9 @@ class ServerRequestHandler:
return True
ds = []
for query_handler, query_identifiers in self.query_handlers.items():
for query_handler, query_identifiers in self.query_handlers.iteritems():
queries = {q_i: msg[q_i] for q_i in query_identifiers if q_i in msg}
d = defer.maybeDeferred(query_handler.handle_queries(queries))
d = query_handler.handle_queries(queries)
d.addErrback(log_errors)
ds.append(d)
File renamed without changes.
+50
View File
@@ -0,0 +1,50 @@
import platform
import json
import subprocess
import os
from urllib2 import urlopen, URLError
from lbryschema import __version__ as lbryschema_version
from lbryum import __version__ as LBRYUM_VERSION
from lbrynet import build_type, __version__ as lbrynet_version
from lbrynet.conf import ROOT_DIR
def get_lbrynet_version():
if build_type.BUILD == "dev":
try:
with open(os.devnull, 'w') as devnull:
git_dir = ROOT_DIR + '/.git'
return subprocess.check_output(
['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'],
stderr=devnull
).strip().lstrip('v')
except (subprocess.CalledProcessError, OSError):
print "failed to get version from git"
return lbrynet_version
def get_platform(get_ip=True):
p = {
"processor": platform.processor(),
"python_version": platform.python_version(),
"platform": platform.platform(),
"os_release": platform.release(),
"os_system": platform.system(),
"lbrynet_version": get_lbrynet_version(),
"lbryum_version": LBRYUM_VERSION,
"lbryschema_version": lbryschema_version,
"build": build_type.BUILD, # CI server sets this during build step
}
# TODO: remove this from get_platform and add a get_external_ip function using txrequests
if get_ip:
try:
response = json.loads(urlopen("https://api.lbry.io/ip").read())
if not response['success']:
raise URLError("failed to get external ip")
p['ip'] = response['data']['ip']
except (URLError, AssertionError):
p['ip'] = "Could not determine IP"
return p
+30 -96
View File
@@ -1,18 +1,18 @@
import base64
import codecs
import datetime
import logging
import random
import socket
import string
import json
import traceback
import functools
import logging
import pkg_resources
from twisted.python.failure import Failure
from twisted.internet import defer
from lbrynet.schema.claim import ClaimDict
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbryschema.claim import ClaimDict
from lbrynet.core.cryptoutils import get_lbry_hash_obj
# digest_size is in bytes, and blob hashes are hex encoded
blobhash_length = get_lbry_hash_obj().digest_size * 2
log = logging.getLogger(__name__)
@@ -49,7 +49,6 @@ def call_later(delay, func, *args, **kwargs):
from twisted.internet import reactor
return reactor.callLater(delay, func, *args, **kwargs)
def safe_start_looping_call(looping_call, interval_sec):
if not looping_call.running:
looping_call.start(interval_sec)
@@ -61,36 +60,46 @@ def safe_stop_looping_call(looping_call):
def generate_id(num=None):
h = get_lbry_hash_obj()
if num is not None:
h.update(str(num).encode())
h.update(str(num))
else:
h.update(str(random.getrandbits(512)).encode())
h.update(str(random.getrandbits(512)))
return h.digest()
def is_valid_hashcharacter(char):
return char in "0123456789abcdef"
def is_valid_blobhash(blobhash):
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
@param blobhash: string, the blobhash to check
@return: True/False
"""
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
def version_is_greater_than(a, b):
"""Returns True if version a is more recent than version b"""
return pkg_resources.parse_version(a) > pkg_resources.parse_version(b)
def rot13(some_str):
return codecs.encode(some_str, 'rot_13')
def deobfuscate(obfustacated):
return base64.b64decode(rot13(obfustacated)).decode()
return base64.b64decode(obfustacated.decode('rot13'))
def obfuscate(plain):
return rot13(base64.b64encode(plain).decode())
return base64.b64encode(plain).encode('rot13')
def check_connection(server="lbry.io", port=80, timeout=5):
def check_connection(server="lbry.io", port=80, timeout=2):
"""Attempts to open a socket to server:port and returns True if successful."""
log.debug('Checking connection to %s:%s', server, port)
try:
server = socket.gethostbyname(server)
conn = socket.create_connection((server, port), timeout)
conn.close()
socket.create_connection((server, port), timeout)
log.debug('Connection successful')
return True
except (socket.gaierror, socket.herror) as ex:
@@ -131,7 +140,7 @@ def get_sd_hash(stream_info):
get('source', {}).\
get('source')
if not result:
log.warning("Unable to get sd_hash")
log.warn("Unable to get sd_hash")
return result
@@ -139,23 +148,12 @@ def json_dumps_pretty(obj, **kwargs):
return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs)
class DeferredLockContextManager:
def __init__(self, lock):
self._lock = lock
def __enter__(self):
yield self._lock.acquire()
def __exit__(self, exc_type, exc_val, exc_tb):
yield self._lock.release()
@defer.inlineCallbacks
def DeferredDict(d, consumeErrors=False):
keys = []
dl = []
response = {}
for k, v in d.items():
for k, v in d.iteritems():
keys.append(k)
dl.append(v)
results = yield defer.DeferredList(dl, consumeErrors=consumeErrors)
@@ -163,67 +161,3 @@ def DeferredDict(d, consumeErrors=False):
if success:
response[k] = result
defer.returnValue(response)
class DeferredProfiler:
def __init__(self):
self.profile_results = {}
def add_result(self, fn, start_time, finished_time, stack, success):
self.profile_results[fn].append((start_time, finished_time, stack, success))
def show_profile_results(self, fn):
profile_results = list(self.profile_results[fn])
call_counts = {
caller: [(start, finished, finished - start, success)
for (start, finished, _caller, success) in profile_results
if _caller == caller]
for caller in {result[2] for result in profile_results}
}
log.info("called %s %i times from %i sources\n", fn.__name__, len(profile_results), len(call_counts))
for caller in sorted(list(call_counts.keys()), key=lambda c: len(call_counts[c]), reverse=True):
call_info = call_counts[caller]
times = [r[2] for r in call_info]
own_time = sum(times)
times.sort()
longest = 0 if not times else times[-1]
shortest = 0 if not times else times[0]
log.info(
"%i successes and %i failures\nlongest %f, shortest %f, avg %f\ncaller:\n%s",
len([r for r in call_info if r[3]]),
len([r for r in call_info if not r[3]]),
longest, shortest, own_time / float(len(call_info)), caller
)
def profiled_deferred(self, reactor=None):
if not reactor:
from twisted.internet import reactor
def _cb(result, fn, start, caller_info):
got_error = isinstance(result, (Failure, Exception))
self.add_result(fn, start, reactor.seconds(), caller_info, not got_error)
if got_error:
raise result
else:
return result
def _profiled_deferred(fn):
reactor.addSystemEventTrigger("after", "shutdown", self.show_profile_results, fn)
self.profile_results[fn] = []
@functools.wraps(fn)
def _wrapper(*args, **kwargs):
caller_info = "".join(traceback.format_list(traceback.extract_stack()[-3:-1]))
start = reactor.seconds()
d = defer.maybeDeferred(fn, *args, **kwargs)
d.addBoth(_cb, fn, start, caller_info)
return d
return _wrapper
return _profiled_deferred
_profiler = DeferredProfiler()
profile_deferred = _profiler.profiled_deferred
@@ -7,7 +7,7 @@ from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
from cryptography.hazmat.backends import default_backend
from lbrynet.p2p.BlobInfo import BlobInfo
from lbrynet.core.BlobInfo import BlobInfo
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
log = logging.getLogger(__name__)
@@ -16,21 +16,21 @@ backend = default_backend()
class CryptBlobInfo(BlobInfo):
def __init__(self, blob_hash, blob_num, length, iv):
super().__init__(blob_hash, blob_num, length)
BlobInfo.__init__(self, blob_hash, blob_num, length)
self.iv = iv
def get_dict(self):
info = {
"blob_num": self.blob_num,
"length": self.length,
"iv": self.iv.decode()
"iv": self.iv
}
if self.blob_hash:
info['blob_hash'] = self.blob_hash
return info
class StreamBlobDecryptor:
class StreamBlobDecryptor(object):
def __init__(self, blob, key, iv, length):
"""
This class decrypts blob
@@ -53,10 +53,10 @@ class StreamBlobDecryptor:
def decrypt(self, write_func):
"""
Decrypt blob and write its content using write_func
Decrypt blob and write its content useing write_func
write_func - function that takes decrypted string as
argument and writes it somewhere
arugment and writes it somewhere
Returns:
@@ -68,14 +68,14 @@ class StreamBlobDecryptor:
def write_bytes():
if self.len_read < self.length:
num_bytes_to_decrypt = greatest_multiple(len(self.buff), (AES.block_size // 8))
num_bytes_to_decrypt = greatest_multiple(len(self.buff), (AES.block_size / 8))
data_to_decrypt, self.buff = split(self.buff, num_bytes_to_decrypt)
write_func(self.cipher.update(data_to_decrypt))
def finish_decrypt():
bytes_left = len(self.buff) % (AES.block_size // 8)
bytes_left = len(self.buff) % (AES.block_size / 8)
if bytes_left != 0:
log.warning(binascii.hexlify(self.buff[-1 * (AES.block_size // 8):]).decode())
log.warning(self.buff[-1 * (AES.block_size / 8):].encode('hex'))
raise Exception("blob %s has incorrect padding: %i bytes left" %
(self.blob.blob_hash, bytes_left))
data_to_decrypt, self.buff = self.buff, b''
@@ -99,7 +99,7 @@ class StreamBlobDecryptor:
return d
class CryptStreamBlobMaker:
class CryptStreamBlobMaker(object):
def __init__(self, key, iv, blob_num, blob):
"""
This class encrypts data and writes it to a new blob
@@ -146,7 +146,7 @@ class CryptStreamBlobMaker:
def close(self):
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
if self.length != 0:
self.length += (AES.block_size // 8) - (self.length % (AES.block_size // 8))
self.length += (AES.block_size / 8) - (self.length % (AES.block_size / 8))
padded_data = self.padder.finalize()
encrypted_data = self.cipher.update(padded_data) + self.cipher.finalize()
self.blob.write(encrypted_data)
@@ -1,18 +1,19 @@
"""
Utility for creating Crypt Streams, which are encrypted blobs and associated metadata.
"""
import os
import logging
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from twisted.internet import defer
from lbrynet.blob.CryptBlob import CryptStreamBlobMaker
import logging
from twisted.internet import interfaces, defer
from zope.interface import implements
from Crypto import Random
from Crypto.Cipher import AES
from lbrynet.cryptstream.CryptBlob import CryptStreamBlobMaker
log = logging.getLogger(__name__)
class CryptStreamCreator:
class CryptStreamCreator(object):
"""
Create a new stream with blobs encrypted by a symmetric cipher.
@@ -21,7 +22,7 @@ class CryptStreamCreator:
the blob is associated with the stream.
"""
#implements(interfaces.IConsumer)
implements(interfaces.IConsumer)
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
"""@param blob_manager: Object that stores and provides access to blobs.
@@ -100,13 +101,13 @@ class CryptStreamCreator:
@staticmethod
def random_iv_generator():
while 1:
yield os.urandom(AES.block_size // 8)
yield Random.new().read(AES.block_size)
def setup(self):
"""Create the symmetric key if it wasn't provided"""
if self.key is None:
self.key = os.urandom(AES.block_size // 8)
self.key = Random.new().read(AES.block_size)
return defer.succeed(True)
@@ -121,7 +122,7 @@ class CryptStreamCreator:
yield defer.DeferredList(self.finished_deferreds)
self.blob_count += 1
iv = next(self.iv_generator)
iv = self.iv_generator.next()
final_blob = self._get_blob_maker(iv, self.blob_manager.get_blob_creator())
stream_terminator = yield final_blob.close()
terminator_info = yield self._blob_finished(stream_terminator)
@@ -132,7 +133,7 @@ class CryptStreamCreator:
if self.current_blob is None:
self.next_blob_creator = self.blob_manager.get_blob_creator()
self.blob_count += 1
iv = next(self.iv_generator)
iv = self.iv_generator.next()
self.current_blob = self._get_blob_maker(iv, self.next_blob_creator)
done, num_bytes_written = self.current_blob.write(data)
data = data[num_bytes_written:]
+8
View File
@@ -0,0 +1,8 @@
"""
Classes and functions for dealing with Crypt Streams.
Crypt Streams are encrypted blobs and metadata tying those blobs together. At least some of the
metadata is generally stored in a Stream Descriptor File, for example containing a public key
used to bind blobs to the stream and a symmetric key used to encrypt the blobs. The list of blobs
may or may not be present.
"""
@@ -1,10 +1,12 @@
import binascii
from zope.interface import implements
from twisted.internet import defer
from lbrynet.blob.CryptBlob import StreamBlobDecryptor
from lbrynet.cryptstream.CryptBlob import StreamBlobDecryptor
from lbrynet.interfaces import IBlobHandler
class CryptBlobHandler:
#implements(IBlobHandler)
class CryptBlobHandler(object):
implements(IBlobHandler)
def __init__(self, key, write_func):
self.key = key
@@ -1,15 +1,15 @@
import binascii
import logging
from binascii import unhexlify
from zope.interface import implements
from lbrynet.interfaces import IStreamDownloader
from lbrynet.core.client.BlobRequester import BlobRequester
from lbrynet.core.client.ConnectionManager import ConnectionManager
from lbrynet.core.client.DownloadManager import DownloadManager
from lbrynet.core.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.cryptstream.client.CryptBlobHandler import CryptBlobHandler
from twisted.internet import defer
from twisted.python.failure import Failure
from lbrynet.conf import Config
from lbrynet.p2p.client.BlobRequester import BlobRequester
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
from lbrynet.p2p.client.DownloadManager import DownloadManager
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler
log = logging.getLogger(__name__)
@@ -34,11 +34,11 @@ class CurrentlyStartingError(Exception):
pass
class CryptStreamDownloader:
class CryptStreamDownloader(object):
#implements(IStreamDownloader)
implements(IStreamDownloader)
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
def __init__(self, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
key, stream_name):
"""Initialize a CryptStreamDownloader
@@ -56,14 +56,14 @@ class CryptStreamDownloader:
@return:
"""
self.conf = conf
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.blob_manager = blob_manager
self.payment_rate_manager = payment_rate_manager
self.wallet = wallet
self.key = unhexlify(key)
self.stream_name = unhexlify(stream_name).decode()
self.key = binascii.unhexlify(key)
self.stream_name = binascii.unhexlify(stream_name)
self.completed = False
self.stopped = True
self.stopping = False
File diff suppressed because it is too large. Load diff
Loaded 100 of 357 files, more files were not shown because too many files have changed in this diff. Show more