Compare commits

..
2 Commits
384 changed files with 3962 additions and 37442 deletions

No files matched your search

+1 -1
View File
@@ -2,6 +2,6 @@
.tox
__pycache__
dist
lbry.egg-info
lbrynet.egg-info
docs
tests
File renamed without changes.
+74
View File
@@ -0,0 +1,74 @@
<!--
Thanks for reporting an issue to LBRY and helping us improve!
To make it possible for us to help you, please fill out below information carefully.
Before reporting any issues, please make sure that you're using the latest version.
- App: https://github.com/lbryio/lbry-desktop/releases
- Daemon: https://github.com/lbryio/lbry/releases
We are also available on Discord at https://chat.lbry.io
-->
## The Issue
In order to <achieve some value>,
as a <type of user>,
I want <some functionality>.
### Steps to reproduce
1.
2.
3.
### Expected behaviour
Tell us what should happen
### Actual behaviour
Tell us what happens instead
## System Configuration
<!-- For the app, this info is in the About section at the bottom of the Help page.
You can include a screenshot instead of typing it out -->
<!-- For the daemon, run:
curl 'http://localhost:5279' --data '{"method":"version"}'
and include the full output -->
- LBRY Daemon version:
- LBRY App version:
- LBRY Installation ID:
- Operating system:
## Anything Else
<!-- Include anything else that does not fit into the above sections -->
## Screenshots
<!-- If a screenshot would help explain the bug, please include one or two here -->
## Internal Use
### Acceptance Criteria
1.
2.
3.
### Definition of Done
- [ ] Tested against acceptance criteria
- [ ] Tested against the assumptions of user story
- [ ] The project builds without errors
- [ ] Unit tests are written and passing
- [ ] Tests on devices/browsers listed in the issue have passed
- [ ] QA performed & issues resolved
- [ ] Refactoring completed
- [ ] Any configuration or build changes documented
- [ ] Documentation updated
- [ ] Peer Code Review performed
+38
View File
@@ -0,0 +1,38 @@
## PR Checklist
Please check all that apply to this PR using "x":
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
- [ ] I have checked that this PR does not introduce a breaking change
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
## PR Type
What kind of change does this PR introduce?
Why is this change necessary?
<!-- Please check all that apply to this PR using "x". -->
- [ ] Bugfix
- [ ] Feature
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
- [ ] Code style update (formatting)
- [ ] Refactoring (no functional changes)
- [ ] Documentation changes
- [ ] Other - Please describe:
## Fixes
Issue Number: N/A
## What is the current behavior?
## What is the new behavior?
## Other information
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
+12
View File
@@ -1 +1,13 @@
.DS_Store
/build
/dist
/.tox
/.idea
/.coverage
/lbry-venv
lbrynet.egg-info
__pycache__
_trial_temp/
/tests/integration/files
+5 -1
View File
@@ -24,7 +24,7 @@ persistent=yes
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=4
jobs=1
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
@@ -77,6 +77,8 @@ disable=
dangerous-default-value,
duplicate-code,
fixme,
global-statement,
inherit-non-class,
invalid-name,
len-as-condition,
locally-disabled,
@@ -109,11 +111,13 @@ disable=
unnecessary-lambda,
unused-argument,
unused-variable,
wildcard-import,
wrong-import-order,
wrong-import-position,
deprecated-lambda,
simplifiable-if-statement,
unidiomatic-typecheck,
global-at-module-level,
inconsistent-return-statements,
keyword-arg-before-vararg,
assignment-from-no-return,
+77 -84
View File
@@ -7,140 +7,133 @@ jobs:
include:
- stage: code quality
name: "pylint & mypy"
name: "pylint lbrynet"
install:
- make install
script: make lint
- pip install astroid pylint
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .
script: pylint lbrynet
- stage: test
name: "LBRY Unit Tests"
name: "Unit Tests"
install:
- make install
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .
script:
- cd lbry && HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
after_success:
- coverage combine lbry/
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- name: "LBRY Integration Tests"
- name: "Integration Tests"
install:
- pip install coverage tox-travis
script: cd lbry && tox
- pip install tox-travis coverage
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
script: tox
after_success:
- coverage combine lbry
- coverage combine tests/
- bash <(curl -s https://codecov.io/bash)
- &torba-tests
name: "Torba Unit Tests"
env: TESTTYPE=unit
install:
- pip install coverage tox-travis
script: cd torba && tox
after_success:
- coverage combine torba/tests
- bash <(curl -s https://codecov.io/bash)
- <<: *torba-tests
name: "Torba Integration Tests"
env: TESTTYPE=integration
- name: "Run Examples"
install:
- make install
script:
- cd lbry && HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
after_success:
- coverage combine lbry
- bash <(curl -s https://codecov.io/bash)
- &build
stage: build
name: "Linux"
env: OS=linux
install:
- pip install pyinstaller awscli
- cd torba && pip install -e . && cd ..
- cd lbry
- python scripts/set_build.py
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .
script:
- pyinstaller -F -n lbrynet lbry/extras/cli.py
- cd dist
- chmod +x lbrynet
- zip -j lbrynet-${OS}.zip lbrynet
- shasum -a 256 -b lbrynet-${OS}.zip
- ./lbrynet --version
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
after_success:
- aws configure set aws_access_key_id $ARTIFACTS_KEY
- aws configure set aws_secret_access_key $ARTIFACTS_SECRET
- aws configure set region us-east-1
- export S3_PATH="daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})"
- aws s3 cp lbrynet-${OS}.zip s3://build.lbry.io/${S3_PATH}/lbrynet-${OS}.zip
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- stage: build
name: "Windows"
language: generic
services:
- docker
install:
- docker pull lbry/pyinstaller34_32bits:py371
script:
- python scripts/set_build.py
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
file: lbrynet-${OS}.zip
file: dist/lbrynet-windows.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
paths:
- lbrynet-windows.zip
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- &build
name: "Linux"
env: OS=linux
install:
- pip3 install pyinstaller
- pip3 install git+https://github.com/lbryio/torba.git
- python3 scripts/set_build.py
- pip3 install -e .
script:
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
- chmod +x dist/lbrynet
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
- ./dist/lbrynet --version
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
file: dist/lbrynet-${OS}.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
paths:
- lbrynet-${OS}.zip
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- <<: *build
name: "Mac"
os: osx
osx_image: xcode8.3
language: shell
language: generic
env: OS=mac
cache: false
before_install:
- brew upgrade python || true
- pip3 install --user --upgrade pip virtualenv
- /Users/travis/Library/Python/3.7/bin/virtualenv $HOME/venv
- source $HOME/venv/bin/activate
before_cache:
- brew cleanup
- <<: *build
name: "Windows"
os: windows
language: shell
env:
- OS=windows
- PATH=/C/Users/travis/AppData/Roaming/Python/Python37/Scripts:/c/Python37:/c/Python37/Scripts:/C/Windows/System32/downlevel:$PATH
before_install:
- choco install python --x86
- pip install --user --upgrade pip virtualenv
- virtualenv $HOME/venv
- source $HOME/venv/Scripts/activate
- pip install pywin32
script:
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
- cd dist
- 7z a -tzip lbrynet-windows.zip lbrynet.exe
- sha256sum -b lbrynet-windows.zip
- ./lbrynet.exe --version
- brew upgrade python || true
- if: tag IS present
stage: build
name: "Wallet Server Docker Image - Tagged Release"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f lbry/scripts/Dockerfile.wallet_server .
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
- docker push lbry/wallet-server:$TRAVIS_TAG
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
stage: build
name: "Wallet Server Docker Image - Master"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:master -f lbry/scripts/Dockerfile.wallet_server .
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
- docker push lbry/wallet-server:master
cache:
directories:
- $HOME/venv
- $HOME/.cache/pip
- $HOME/Library/Caches/pip
- $HOME/Library/Caches/Homebrew
- $TRAVIS_BUILD_DIR/.tox
+6
View File
@@ -0,0 +1,6 @@
# Add a CHANGELOG entry for app changes
has_app_changes = !git.modified_files.grep(/lbrynet/).empty?
if !git.modified_files.include?("CHANGELOG.md") && has_app_changes
fail("Please include a CHANGELOG entry.")
message "See http://keepachangelog.com/en/0.3.0/ for details on good changelog guidelines"
end
+16 -51
View File
@@ -9,7 +9,7 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
## Prerequisites
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred). Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
After installing python 3, you'll need to install some additional libraries depending on your operating system.
@@ -17,10 +17,6 @@ After installing python 3, you'll need to install some additional libraries depe
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
These environment variables also need to be set
1. PYTHONUNBUFFERED=1
2. EVENT_NOKQUEUE=1
Remaining dependencies can then be installed by running:
```
@@ -31,12 +27,10 @@ Assistance installing Python3: https://docs.python-guide.org/starting/install3/o
### Linux
On Ubuntu (16.04 minimum, we recommend 18.04), install the following:
On Ubuntu (we recommend 18.04), install the following:
```
sudo add-apt-repository ppa:deadsnakes/ppa
sudo apt-get update
sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
```
On Raspbian, you will also need to install `python-pyparsing`.
@@ -45,61 +39,32 @@ If you're running another Linux distro, install the equivalent of the above pack
## Installation
### Linux/Mac
To install on Linux/Mac:
To install:
```
Clone the repository:
$ git clone https://github.com/lbryio/lbry-sdk.git
$ cd lbry-sdk
git clone https://github.com/lbryio/lbry.git
cd lbry
Create a Python virtual environment for lbry-sdk:
$ python3.7 -m venv lbry-venv
Creating venv:
python -m venv lbry-venv
Activating lbry-sdk virtual environment:
$ source lbry-venv/bin/activate
Activating lbry-venv on Linux/Mac:
source lbry-venv/bin/activate
Make sure you're on Python 3.7+ (as the default Python in virtual environment):
$ python --version
Activating lbry-venv on Windows:
lbry-venv\Scripts\activate
Install packages:
$ make install
python --version # Python 2 is not supported. Make sure you're on Python 3.7
If you are on Linux and using PyCharm, generates initial configs:
$ make idea
pip install -e .
```
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `python3.7 -m venv lbry-venv` command.
### Windows
To install on Windows:
```
Clone the repository:
> git clone https://github.com/lbryio/lbry-sdk.git
> cd lbry-sdk
Create a Python virtual environment for lbry-sdk:
> python -m venv lbry-venv
Activating lbry-sdk virtual environment:
> lbry-venv\Scripts\activate
Install packages:
> cd torba
> pip install -e .
> cd ../lbry
> pip install -e .
```
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
## Run the tests
To run the unit tests from the repo directory:
```
python -m unittest discover -s lbry tests.unit
trial --reactor=asyncio tests.unit
```
## Usage
-14
View File
@@ -1,14 +0,0 @@
install:
cd torba && pip install -e .
cd lbry && pip install -e .
pip install mypy==0.701
pip install coverage astroid pylint
lint:
cd lbry && pylint lbry
cd torba && pylint --rcfile=setup.cfg torba
cd torba && mypy --ignore-missing-imports torba
idea:
mkdir -p .idea
cp -r lbry/scripts/idea/* .idea
+9 -9
View File
@@ -1,19 +1,19 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry-sdk.svg?branch=master)](https://travis-ci.org/lbryio/lbry-sdk) [![Test Coverage](https://codecov.io/gh/lbryio/lbry-sdk/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry-sdk)
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
* Built on Python 3.7+ and `asyncio`.
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/schema)).
* Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/extras/daemon)).
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
* Wallet implementation for the LBRY blockchain ([lbrynet.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
## Installation
Our [releases page](https://github.com/lbryio/lbry-sdk/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
## Usage
@@ -41,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
## Contact
@@ -53,4 +53,4 @@ The documentation for the API can be found [here](https://lbry.tech/api/sdk).
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/lbry/example_daemon_settings.yml).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry/blob/master/example_daemon_settings.yml).
File renamed without changes.
File diff suppressed because it is too large. Load diff
File renamed without changes.

Before

Width:  |  Height:  |  Size: 521 B

After

Width:  |  Height:  |  Size: 521 B

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.4 KiB

After

Width:  |  Height:  |  Size: 1.4 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.6 KiB

After

Width:  |  Height:  |  Size: 1.6 KiB

File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -4,9 +4,8 @@
share_usage_data: True
lbryum_servers:
- lbryumx1.lbry.com:50001
- lbryumx2.lbry.com:50001
- lbryumx4.lbry.com:50001
- lbryumx1.lbry.io:50001
- lbryumx2.lbry.io:50001
blockchain_name: lbrycrd_main
File renamed without changes.

Before

Width:  |  Height:  |  Size: 7.4 KiB

After

Width:  |  Height:  |  Size: 7.4 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 6.1 KiB

After

Width:  |  Height:  |  Size: 6.1 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 97 KiB

After

Width:  |  Height:  |  Size: 97 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 361 KiB

After

Width:  |  Height:  |  Size: 361 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 5.3 KiB

After

Width:  |  Height:  |  Size: 5.3 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 31 KiB

After

Width:  |  Height:  |  Size: 31 KiB

View File
File renamed without changes.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

-13
View File
@@ -1,13 +0,0 @@
.DS_Store
/build
/dist
/.tox
/.idea
/.coverage
/lbry-venv
lbry.egg-info
__pycache__
_trial_temp/
/tests/integration/files
-56
View File
@@ -1,56 +0,0 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry-sdk.svg?branch=master)](https://travis-ci.org/lbryio/lbry-sdk) [![Test Coverage](https://codecov.io/gh/lbryio/lbry-sdk/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry-sdk)
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
* Built on Python 3.7+ and `asyncio`.
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/schema)).
* Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/extras/daemon)).
## Installation
Our [releases page](https://github.com/lbryio/lbry-sdk/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
## Usage
Run `lbrynet start` to launch the API server.
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
With the daemon running, `lbrynet commands` will show you a list of commands.
The full API is documented [here](https://lbry.tech/api/sdk).
## Running from source
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
## Contributing
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
## License
This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
## Contact
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
## Additional information and links
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/lbry/example_daemon_settings.yml).
-4
View File
@@ -1,4 +0,0 @@
__name__ = "lbry"
__version__ = "0.38.0"
version = tuple(__version__.split('.'))
-90
View File
@@ -1,90 +0,0 @@
import asyncio
import typing
import collections
import logging
log = logging.getLogger(__name__)
CONNECTED_EVENT = "connected"
DISCONNECTED_EVENT = "disconnected"
TRANSFERRED_EVENT = "transferred"
class ConnectionManager:
def __init__(self, loop: asyncio.AbstractEventLoop):
self.loop = loop
self.incoming_connected: typing.Set[str] = set()
self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int)
self.outgoing_connected: typing.Set[str] = set()
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
self._status = {}
self._task: typing.Optional[asyncio.Task] = None
@property
def status(self):
return self._status
def sent_data(self, host_and_port: str, size: int):
self.outgoing[host_and_port] += size
def received_data(self, host_and_port: str, size: int):
self.incoming[host_and_port] += size
def connection_made(self, host_and_port: str):
self.outgoing_connected.add(host_and_port)
def connection_received(self, host_and_port: str):
# self.incoming_connected.add(host_and_port)
pass
def outgoing_connection_lost(self, host_and_port: str):
if host_and_port in self.outgoing_connected:
self.outgoing_connected.remove(host_and_port)
def incoming_connection_lost(self, host_and_port: str):
if host_and_port in self.incoming_connected:
self.incoming_connected.remove(host_and_port)
async def _update(self):
self._status = {
'incoming_bps': {},
'outgoing_bps': {},
'total_incoming_mbs': 0.0,
'total_outgoing_mbs': 0.0,
'time': self.loop.time()
}
while True:
last = self.loop.time()
await asyncio.sleep(1, loop=self.loop)
self._status['incoming_bps'].clear()
self._status['outgoing_bps'].clear()
while self.outgoing:
k, v = self.outgoing.popitem()
self._status['outgoing_bps'][k] = v
while self.incoming:
k, v = self.incoming.popitem()
self._status['incoming_bps'][k] = v
now = self.loop.time()
self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values())
) / (now - last)) / 1000000.0
self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values())
) / (now - last)) / 1000000.0
self._status['time'] = now
def stop(self):
if self._task:
self._task.cancel()
self._task = None
self.outgoing.clear()
self.outgoing_connected.clear()
self.incoming.clear()
self.incoming_connected.clear()
self._status.clear()
def start(self):
self.stop()
self._task = self.loop.create_task(self._update())
-70
View File
@@ -1,70 +0,0 @@
import asyncio
import typing
from lbry.dht import constants
if typing.TYPE_CHECKING:
from lbry.dht.peer import KademliaPeer, PeerManager
class DictDataStore:
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager'):
# Dictionary format:
# { <key>: [(<contact>, <age>), ...] }
self._data_store: typing.Dict[bytes, typing.List[typing.Tuple['KademliaPeer', float]]] = {}
self.loop = loop
self._peer_manager = peer_manager
self.completed_blobs: typing.Set[str] = set()
def removed_expired_peers(self):
now = self.loop.time()
keys = list(self._data_store.keys())
for key in keys:
to_remove = []
for (peer, ts) in self._data_store[key]:
if ts + constants.data_expiration < now or self._peer_manager.peer_is_good(peer) is False:
to_remove.append((peer, ts))
for item in to_remove:
self._data_store[key].remove(item)
if not self._data_store[key]:
del self._data_store[key]
def filter_bad_and_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
"""
Returns only non-expired and unknown/good peers
"""
for peer in self.filter_expired_peers(key):
if self._peer_manager.peer_is_good(peer) is not False:
yield peer
def filter_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
"""
Returns only non-expired peers
"""
now = self.loop.time()
for (peer, ts) in self._data_store.get(key, []):
if ts + constants.data_expiration > now:
yield peer
def has_peers_for_blob(self, key: bytes) -> bool:
return key in self._data_store
def add_peer_to_blob(self, contact: 'KademliaPeer', key: bytes) -> None:
now = self.loop.time()
if key in self._data_store:
current = list(filter(lambda x: x[0] == contact, self._data_store[key]))
if len(current):
self._data_store[key][self._data_store[key].index(current[0])] = contact, now
else:
self._data_store[key].append((contact, now))
else:
self._data_store[key] = [(contact, now)]
def get_peers_for_blob(self, key: bytes) -> typing.List['KademliaPeer']:
return list(self.filter_bad_and_expired_peers(key))
def get_storing_contacts(self) -> typing.List['KademliaPeer']:
peers = set()
for key, stored in self._data_store.items():
peers.update(set(map(lambda tup: tup[0], stored)))
return list(peers)
-1
View File
@@ -1 +0,0 @@
from lbry.extras.daemon import Components # register Component classes
-56
View File
@@ -1,56 +0,0 @@
import logging
import time
import hashlib
import binascii
import aiohttp
import ecdsa
from torba.client.hash import sha256
from lbry.wallet.transaction import Output
log = logging.getLogger(__name__)
def get_encoded_signature(signature):
signature = signature.encode() if type(signature) is str else signature
r = int(signature[:int(len(signature) / 2)], 16)
s = int(signature[int(len(signature) / 2):], 16)
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
def is_comment_signed_by_channel(comment: dict, channel: Output):
try:
pieces = [
comment['signing_ts'].encode(),
channel.claim_hash,
comment['comment'].encode()
]
return Output.is_signature_valid(
get_encoded_signature(comment['signature']),
sha256(b''.join(pieces)),
channel.claim.channel.public_key_bytes
)
except KeyError:
pass
return False
def sign_comment(comment: dict, channel: Output):
timestamp = str(int(time.time())).encode()
pieces = [timestamp, channel.claim_hash, comment['comment'].encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
comment['signature'] = binascii.hexlify(signature).decode()
comment['signing_ts'] = timestamp.decode()
async def jsonrpc_post(url: str, method: str, **params) -> any:
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
headers = {'Content-Type': 'application/json'}
async with aiohttp.request('POST', url, json=json_body, headers=headers) as response:
try:
result = await response.json()
return result['result'] if 'result' in result else result
except aiohttp.client.ContentTypeError as cte:
log.exception('Unable to decode respose from server: %s', cte)
return await response.text()
-120
View File
@@ -1,120 +0,0 @@
import base64
import struct
from typing import List
from binascii import hexlify
from itertools import chain
from lbry.schema.types.v2.result_pb2 import Outputs as OutputsMessage
class Outputs:
__slots__ = 'txos', 'extra_txos', 'txs', 'offset', 'total'
def __init__(self, txos: List, extra_txos: List, txs: set, offset: int, total: int):
self.txos = txos
self.txs = txs
self.extra_txos = extra_txos
self.offset = offset
self.total = total
def inflate(self, txs):
tx_map = {tx.hash: tx for tx in txs}
for txo_message in self.extra_txos:
self.message_to_txo(txo_message, tx_map)
return [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos]
def message_to_txo(self, txo_message, tx_map):
if txo_message.WhichOneof('meta') == 'error':
return None
txo = tx_map[txo_message.tx_hash].outputs[txo_message.nout]
if txo_message.WhichOneof('meta') == 'claim':
claim = txo_message.claim
txo.meta = {
'short_url': f'lbry://{claim.short_url}',
'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}',
'is_controlling': claim.is_controlling,
'take_over_height': claim.take_over_height,
'creation_height': claim.creation_height,
'activation_height': claim.activation_height,
'expiration_height': claim.expiration_height,
'effective_amount': claim.effective_amount,
'support_amount': claim.support_amount,
'trending_group': claim.trending_group,
'trending_mixed': claim.trending_mixed,
'trending_local': claim.trending_local,
'trending_global': claim.trending_global,
}
if claim.HasField('channel'):
txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
try:
if txo.claim.is_channel:
txo.meta['claims_in_channel'] = claim.claims_in_channel
except:
pass
return txo
@classmethod
def from_base64(cls, data: str) -> 'Outputs':
return cls.from_bytes(base64.b64decode(data))
@classmethod
def from_bytes(cls, data: bytes) -> 'Outputs':
outputs = OutputsMessage()
outputs.ParseFromString(data)
txs = set()
for txo_message in chain(outputs.txos, outputs.extra_txos):
if txo_message.WhichOneof('meta') == 'error':
continue
txs.add((hexlify(txo_message.tx_hash[::-1]).decode(), txo_message.height))
return cls(outputs.txos, outputs.extra_txos, txs, outputs.offset, outputs.total)
@classmethod
def to_base64(cls, txo_rows, extra_txo_rows, offset=0, total=None) -> str:
return base64.b64encode(cls.to_bytes(txo_rows, extra_txo_rows, offset, total)).decode()
@classmethod
def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None) -> bytes:
page = OutputsMessage()
page.offset = offset
page.total = total or len(txo_rows)
for row in txo_rows:
cls.row_to_message(row, page.txos.add())
for row in extra_txo_rows:
cls.row_to_message(row, page.extra_txos.add())
return page.SerializeToString()
@classmethod
def row_to_message(cls, txo, txo_message):
if isinstance(txo, Exception):
txo_message.error.text = txo.args[0]
if isinstance(txo, ValueError):
txo_message.error.code = txo_message.error.INVALID
elif isinstance(txo, LookupError):
txo_message.error.code = txo_message.error.NOT_FOUND
return
txo_message.tx_hash = txo['txo_hash'][:32]
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
txo_message.height = txo['height']
txo_message.claim.short_url = txo['short_url']
if txo['canonical_url'] is not None:
txo_message.claim.canonical_url = txo['canonical_url']
txo_message.claim.is_controlling = bool(txo['is_controlling'])
if txo['last_take_over_height'] is not None:
txo_message.claim.take_over_height = txo['last_take_over_height']
txo_message.claim.creation_height = txo['creation_height']
txo_message.claim.activation_height = txo['activation_height']
txo_message.claim.expiration_height = txo['expiration_height']
if txo['claims_in_channel'] is not None:
txo_message.claim.claims_in_channel = txo['claims_in_channel']
txo_message.claim.effective_amount = txo['effective_amount']
txo_message.claim.support_amount = txo['support_amount']
txo_message.claim.trending_group = txo['trending_group']
txo_message.claim.trending_mixed = txo['trending_mixed']
txo_message.claim.trending_local = txo['trending_local']
txo_message.claim.trending_global = txo['trending_global']
if txo['channel_txo_hash']:
channel = txo_message.claim.channel
channel.tx_hash = txo['channel_txo_hash'][:32]
channel.nout, = struct.unpack('<I', txo['channel_txo_hash'][32:])
channel.height = txo['channel_height']
-13
View File
@@ -1,13 +0,0 @@
from typing import List
import re
MULTI_SPACE_RE = re.compile(r"\s{2,}")
WEIRD_CHARS_RE = re.compile(r"[#!~]")
def normalize_tag(tag: str):
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower())).strip()
def clean_tags(tags: List[str]):
return [tag for tag in (normalize_tag(tag) for tag in tags) if tag]
-383
View File
@@ -1,383 +0,0 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: result.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='result.proto',
package='pb',
syntax='proto3',
serialized_options=None,
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"b\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\x81\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x02 \x01(\t\x12\x15\n\rcanonical_url\x18\x03 \x01(\t\x12\x16\n\x0eis_controlling\x18\x04 \x01(\x08\x12\x18\n\x10take_over_height\x18\x05 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x06 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x07 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\x08 \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\t \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\n \x01(\x04\x12\x16\n\x0esupport_amount\x18\x0b \x01(\x04\x12\x16\n\x0etrending_group\x18\x0c \x01(\r\x12\x16\n\x0etrending_mixed\x18\r \x01(\x02\x12\x16\n\x0etrending_local\x18\x0e \x01(\x02\x12\x17\n\x0ftrending_global\x18\x0f \x01(\x02\"i\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\"4\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x62\x06proto3')
)
_ERROR_CODE = _descriptor.EnumDescriptor(
name='Code',
full_name='pb.Error.Code',
filename=None,
file=DESCRIPTOR,
values=[
_descriptor.EnumValueDescriptor(
name='UNKNOWN_CODE', index=0, number=0,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='NOT_FOUND', index=1, number=1,
serialized_options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='INVALID', index=2, number=2,
serialized_options=None,
type=None),
],
containing_type=None,
serialized_options=None,
serialized_start=686,
serialized_end=738,
)
_sym_db.RegisterEnumDescriptor(_ERROR_CODE)
_OUTPUTS = _descriptor.Descriptor(
name='Outputs',
full_name='pb.Outputs',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='txos', full_name='pb.Outputs.txos', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='extra_txos', full_name='pb.Outputs.extra_txos', index=1,
number=2, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='total', full_name='pb.Outputs.total', index=2,
number=3, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='offset', full_name='pb.Outputs.offset', index=3,
number=4, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=20,
serialized_end=118,
)
_OUTPUT = _descriptor.Descriptor(
name='Output',
full_name='pb.Output',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='tx_hash', full_name='pb.Output.tx_hash', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='nout', full_name='pb.Output.nout', index=1,
number=2, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='height', full_name='pb.Output.height', index=2,
number=3, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='claim', full_name='pb.Output.claim', index=3,
number=7, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='error', full_name='pb.Output.error', index=4,
number=15, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
_descriptor.OneofDescriptor(
name='meta', full_name='pb.Output.meta',
index=0, containing_type=None, fields=[]),
],
serialized_start=120,
serialized_end=243,
)
_CLAIMMETA = _descriptor.Descriptor(
name='ClaimMeta',
full_name='pb.ClaimMeta',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='channel', full_name='pb.ClaimMeta.channel', index=0,
number=1, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='short_url', full_name='pb.ClaimMeta.short_url', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=3,
number=4, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='take_over_height', full_name='pb.ClaimMeta.take_over_height', index=4,
number=5, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='creation_height', full_name='pb.ClaimMeta.creation_height', index=5,
number=6, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='activation_height', full_name='pb.ClaimMeta.activation_height', index=6,
number=7, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=7,
number=8, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=8,
number=9, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=9,
number=10, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='support_amount', full_name='pb.ClaimMeta.support_amount', index=10,
number=11, type=4, cpp_type=4, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='trending_group', full_name='pb.ClaimMeta.trending_group', index=11,
number=12, type=13, cpp_type=3, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='trending_mixed', full_name='pb.ClaimMeta.trending_mixed', index=12,
number=13, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='trending_local', full_name='pb.ClaimMeta.trending_local', index=13,
number=14, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='trending_global', full_name='pb.ClaimMeta.trending_global', index=14,
number=15, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=246,
serialized_end=631,
)
_ERROR = _descriptor.Descriptor(
name='Error',
full_name='pb.Error',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='code', full_name='pb.Error.code', index=0,
number=1, type=14, cpp_type=8, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='text', full_name='pb.Error.text', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
_ERROR_CODE,
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=633,
serialized_end=738,
)
_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
_OUTPUTS.fields_by_name['extra_txos'].message_type = _OUTPUT
_OUTPUT.fields_by_name['claim'].message_type = _CLAIMMETA
_OUTPUT.fields_by_name['error'].message_type = _ERROR
_OUTPUT.oneofs_by_name['meta'].fields.append(
_OUTPUT.fields_by_name['claim'])
_OUTPUT.fields_by_name['claim'].containing_oneof = _OUTPUT.oneofs_by_name['meta']
_OUTPUT.oneofs_by_name['meta'].fields.append(
_OUTPUT.fields_by_name['error'])
_OUTPUT.fields_by_name['error'].containing_oneof = _OUTPUT.oneofs_by_name['meta']
_CLAIMMETA.fields_by_name['channel'].message_type = _OUTPUT
_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE
_ERROR_CODE.containing_type = _ERROR
DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS
DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT
DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA
DESCRIPTOR.message_types_by_name['Error'] = _ERROR
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Outputs = _reflection.GeneratedProtocolMessageType('Outputs', (_message.Message,), dict(
DESCRIPTOR = _OUTPUTS,
__module__ = 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.Outputs)
))
_sym_db.RegisterMessage(Outputs)
Output = _reflection.GeneratedProtocolMessageType('Output', (_message.Message,), dict(
DESCRIPTOR = _OUTPUT,
__module__ = 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.Output)
))
_sym_db.RegisterMessage(Output)
ClaimMeta = _reflection.GeneratedProtocolMessageType('ClaimMeta', (_message.Message,), dict(
DESCRIPTOR = _CLAIMMETA,
__module__ = 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.ClaimMeta)
))
_sym_db.RegisterMessage(ClaimMeta)
Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), dict(
DESCRIPTOR = _ERROR,
__module__ = 'result_pb2'
# @@protoc_insertion_point(class_scope:pb.Error)
))
_sym_db.RegisterMessage(Error)
# @@protoc_insertion_point(module_scope)
-126
View File
@@ -1,126 +0,0 @@
import re
import unicodedata
from typing import NamedTuple, Tuple
def _create_url_regex():
# see https://spec.lbry.com/
def _named(name, regex):
return "(?P<" + name + ">" + regex + ")"
def _group(regex):
return "(?:" + regex + ")"
def _oneof(*choices):
return _group('|'.join(choices))
def _claim(name, prefix=""):
return _group(
_named(name+"_name", prefix + "[^=&#:$@%?/]+") +
_oneof(
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
) + '?'
)
return (
'^' +
_named("scheme", "lbry://") + '?' +
_oneof(
_group(_claim("channel_with_stream", "@") + "/" + _claim("stream_in_channel")),
_claim("channel", "@"),
_claim("stream")
) +
'$'
)
URL_REGEX = _create_url_regex()
def normalize_name(name):
return unicodedata.normalize('NFD', name).casefold()
class PathSegment(NamedTuple):
name: str
claim_id: str = None
sequence: int = None
amount_order: int = None
@property
def normalized(self):
return normalize_name(self.name)
def to_dict(self):
q = {'name': self.name}
if self.claim_id is not None:
q['claim_id'] = self.claim_id
if self.sequence is not None:
q['sequence'] = self.sequence
if self.amount_order is not None:
q['amount_order'] = self.amount_order
return q
def __str__(self):
if self.claim_id is not None:
return f"{self.name}#{self.claim_id}"
elif self.sequence is not None:
return f"{self.name}:{self.sequence}"
elif self.amount_order is not None:
return f"{self.name}${self.amount_order}"
return self.name
class URL(NamedTuple):
stream: PathSegment
channel: PathSegment
@property
def has_channel(self):
return self.channel is not None
@property
def has_stream(self):
return self.stream is not None
@property
def has_stream_in_channel(self):
return self.has_channel and self.has_stream
@property
def parts(self) -> Tuple:
if self.has_stream_in_channel:
return self.channel, self.stream
if self.has_channel:
return self.channel,
return self.stream,
def __str__(self):
return f"lbry://{'/'.join(str(p) for p in self.parts)}"
@classmethod
def parse(cls, url):
match = re.match(URL_REGEX, url)
if match is None:
raise ValueError('Invalid LBRY URL')
segments = {}
parts = match.groupdict()
for segment in ('channel', 'stream', 'channel_with_stream', 'stream_in_channel'):
if parts[f'{segment}_name'] is not None:
segments[segment] = PathSegment(
parts[f'{segment}_name'],
parts[f'{segment}_claim_id'],
parts[f'{segment}_sequence'],
parts[f'{segment}_amount_order']
)
if 'channel_with_stream' in segments:
segments['channel'] = segments['channel_with_stream']
segments['stream'] = segments['stream_in_channel']
return cls(segments.get('stream', None), segments.get('channel', None))
-11
View File
@@ -1,11 +0,0 @@
__node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.4.0/lbrycrd-linux.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
from lbry.wallet.manager import LbryWalletManager
from lbry.wallet.network import Network
from lbry.wallet.ledger import MainNetLedger, RegTestLedger, TestNetLedger
-138
View File
@@ -1,138 +0,0 @@
import json
import logging
from hashlib import sha256
from string import hexdigits
import ecdsa
from torba.client.baseaccount import BaseAccount, HierarchicalDeterministic
log = logging.getLogger(__name__)
def validate_claim_id(claim_id):
if not len(claim_id) == 40:
raise Exception("Incorrect claimid length: %i" % len(claim_id))
if isinstance(claim_id, bytes):
claim_id = claim_id.decode('utf-8')
if set(claim_id).difference(hexdigits):
raise Exception("Claim id is not hex encoded")
class Account(BaseAccount):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.channel_keys = {}
@property
def hash(self) -> bytes:
h = sha256(json.dumps(self.to_dict(False)).encode())
for cert in sorted(self.channel_keys.keys()):
h.update(cert.encode())
return h.digest()
def apply(self, d: dict):
super().apply(d)
self.channel_keys.update(d.get('certificates', {}))
def add_channel_private_key(self, private_key):
public_key_bytes = private_key.get_verifying_key().to_der()
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
def get_channel_private_key(self, public_key_bytes):
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
private_key_pem = self.channel_keys.get(channel_pubkey_hash)
if private_key_pem:
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
async def maybe_migrate_certificates(self):
if not self.channel_keys:
return
channel_keys = {}
for private_key_pem in self.channel_keys.values():
if not isinstance(private_key_pem, str):
continue
if "-----BEGIN EC PRIVATE KEY-----" not in private_key_pem:
continue
private_key = ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
public_key_der = private_key.get_verifying_key().to_der()
channel_keys[self.ledger.public_key_to_address(public_key_der)] = private_key_pem
self.channel_keys = channel_keys
self.wallet.save()
async def save_max_gap(self):
if issubclass(self.address_generator, HierarchicalDeterministic):
gap = await self.get_max_gap()
self.receiving.gap = max(20, gap['max_receiving_gap'] + 1)
self.change.gap = max(6, gap['max_change_gap'] + 1)
self.wallet.save()
def get_balance(self, confirmations=0, include_claims=False, **constraints):
if not include_claims:
constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
return super().get_balance(confirmations, **constraints)
@classmethod
def get_private_key_from_seed(cls, ledger, seed: str, password: str):
return super().get_private_key_from_seed(
ledger, seed, password or 'lbryum'
)
@classmethod
def from_dict(cls, ledger, wallet, d: dict) -> 'Account':
account = super().from_dict(ledger, wallet, d)
account.channel_keys = d.get('certificates', {})
return account
def to_dict(self, include_channel_keys=True):
d = super().to_dict()
if include_channel_keys:
d['certificates'] = self.channel_keys
return d
async def get_details(self, **kwargs):
details = await super().get_details(**kwargs)
details['certificates'] = len(self.channel_keys)
return details
@staticmethod
def constraint_spending_utxos(constraints):
constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
def get_utxos(self, **constraints):
self.constraint_spending_utxos(constraints)
return super().get_utxos(**constraints)
def get_utxo_count(self, **constraints):
self.constraint_spending_utxos(constraints)
return super().get_utxo_count(**constraints)
def get_claims(self, **constraints):
return self.ledger.db.get_claims(account=self, **constraints)
def get_claim_count(self, **constraints):
return self.ledger.db.get_claim_count(account=self, **constraints)
def get_streams(self, **constraints):
return self.ledger.db.get_streams(account=self, **constraints)
def get_stream_count(self, **constraints):
return self.ledger.db.get_stream_count(account=self, **constraints)
def get_channels(self, **constraints):
return self.ledger.db.get_channels(account=self, **constraints)
def get_channel_count(self, **constraints):
return self.ledger.db.get_channel_count(account=self, **constraints)
def get_supports(self, **constraints):
return self.ledger.db.get_supports(account=self, **constraints)
def get_support_count(self, **constraints):
return self.ledger.db.get_support_count(account=self, **constraints)
async def release_all_outputs(self):
await self.ledger.db.release_all_outputs(self)
-132
View File
@@ -1,132 +0,0 @@
import asyncio
import logging
from binascii import unhexlify
from typing import Tuple, List, Dict
from torba.client.baseledger import BaseLedger
from torba.client.baseaccount import SingleKey
from lbry.schema.result import Outputs
from lbry.schema.url import URL
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet.resolve import Resolver
from lbry.wallet.account import Account
from lbry.wallet.network import Network
from lbry.wallet.database import WalletDatabase
from lbry.wallet.transaction import Transaction, Output
from lbry.wallet.header import Headers, UnvalidatedHeaders
log = logging.getLogger(__name__)
class MainNetLedger(BaseLedger):
name = 'LBRY Credits'
symbol = 'LBC'
network_name = 'mainnet'
headers: Headers
account_class = Account
database_class = WalletDatabase
headers_class = Headers
network_class = Network
transaction_class = Transaction
db: WalletDatabase
secret_prefix = bytes((0x1c,))
pubkey_address_prefix = bytes((0x55,))
script_address_prefix = bytes((0x7a,))
extended_public_key_prefix = unhexlify('0488b21e')
extended_private_key_prefix = unhexlify('0488ade4')
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
genesis_bits = 0x1f00ffff
target_timespan = 150
default_fee_per_byte = 50
default_fee_per_name_char = 200000
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
self.resolver = Resolver(self)
async def _inflate_outputs(self, query):
outputs = Outputs.from_base64(await query)
txs = []
if len(outputs.txs) > 0:
txs = await asyncio.gather(*(self.cache_transaction(*tx) for tx in outputs.txs))
return outputs.inflate(txs), outputs.offset, outputs.total
async def resolve(self, urls):
txos = (await self._inflate_outputs(self.network.resolve(urls)))[0]
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
result = {}
for url, txo in zip(urls, txos):
if txo and URL.parse(url).has_stream_in_channel:
if not txo.channel or not txo.is_signed_by(txo.channel, self):
txo = None
if txo:
result[url] = txo
else:
result[url] = {'error': f'{url} did not resolve to a claim'}
return result
async def claim_search(self, **kwargs) -> Tuple[List, int, int]:
return await self._inflate_outputs(self.network.claim_search(**kwargs))
async def get_claim_by_claim_id(self, claim_id) -> Dict[str, Output]:
for claim in (await self.claim_search(claim_id=claim_id))[0]:
return claim
async def start(self):
await super().start()
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
await self._report_state()
async def _report_state(self):
try:
for account in self.accounts:
balance = dewies_to_lbc(await account.get_balance())
channel_count = await account.get_channel_count()
claim_count = await account.get_claim_count()
if isinstance(account.receiving, SingleKey):
log.info("Loaded single key account %s with %s LBC. "
"%d channels, %d certificates and %d claims",
account.id, balance, channel_count, len(account.channel_keys), claim_count)
else:
total_receiving = len((await account.receiving.get_addresses()))
total_change = len((await account.change.get_addresses()))
log.info("Loaded account %s with %s LBC, %d receiving addresses (gap: %d), "
"%d change addresses (gap: %d), %d channels, %d certificates and %d claims. ",
account.id, balance, total_receiving, account.receiving.gap, total_change,
account.change.gap, channel_count, len(account.channel_keys), claim_count)
except:
log.exception(
'Failed to display wallet state, please file issue '
'for this bug along with the traceback you see below:')
class TestNetLedger(MainNetLedger):
network_name = 'testnet'
pubkey_address_prefix = bytes((111,))
script_address_prefix = bytes((196,))
extended_public_key_prefix = unhexlify('043587cf')
extended_private_key_prefix = unhexlify('04358394')
class RegTestLedger(MainNetLedger):
network_name = 'regtest'
headers_class = UnvalidatedHeaders
pubkey_address_prefix = bytes((111,))
script_address_prefix = bytes((196,))
extended_public_key_prefix = unhexlify('043587cf')
extended_private_key_prefix = unhexlify('04358394')
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
genesis_bits = 0x207fffff
target_timespan = 1
-16
View File
@@ -1,16 +0,0 @@
from torba.client.basenetwork import BaseNetwork
class Network(BaseNetwork):
def get_server_height(self):
return self.rpc('blockchain.block.get_server_height', [])
def get_claims_by_ids(self, claim_ids):
return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids)
def resolve(self, urls):
return self.rpc('blockchain.claimtrie.resolve', urls)
def claim_search(self, **kwargs):
return self.rpc('blockchain.claimtrie.search', kwargs)
-103
View File
@@ -1,103 +0,0 @@
import time
from torba.server.block_processor import BlockProcessor
from lbry.schema.claim import Claim
from lbry.wallet.server.db import SQLDB
class Timer:
def __init__(self, name):
self.name = name
self.total = 0
self.count = 0
self.sub_timers = {}
self._last_start = None
def add_timer(self, name):
if name not in self.sub_timers:
self.sub_timers[name] = Timer(name)
return self.sub_timers[name]
def run(self, func, *args, forward_timer=False, timer_name=None, **kwargs):
t = self.add_timer(timer_name or func.__name__)
t.start()
try:
if forward_timer:
return func(*args, **kwargs, timer=t)
else:
return func(*args, **kwargs)
finally:
t.stop()
def start(self):
self._last_start = time.time()
return self
def stop(self):
self.total += (time.time() - self._last_start)
self.count += 1
self._last_start = None
return self
def show(self, depth=0, height=None):
if depth == 0:
print('='*100)
if height is not None:
print(f'STATISTICS AT HEIGHT {height}')
print('='*100)
else:
print(
f"{' '*depth} {self.total/60:4.2f}mins {self.name}"
# f"{self.total/self.count:.5f}sec/call, "
)
for sub_timer in self.sub_timers.values():
sub_timer.show(depth+1)
if depth == 0:
print('='*100)
class LBRYBlockProcessor(BlockProcessor):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.env.coin.NET == "regtest":
self.prefetcher.polling_delay = 0.5
self.should_validate_signatures = self.env.boolean('VALIDATE_CLAIM_SIGNATURES', False)
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
self.sql: SQLDB = self.db.sql
self.timer = Timer('BlockProcessor')
def advance_blocks(self, blocks):
self.sql.begin()
try:
self.timer.run(super().advance_blocks, blocks)
except:
self.logger.exception(f'Error while advancing transaction in new block.')
raise
finally:
self.sql.commit()
def advance_txs(self, height, txs, header):
timer = self.timer.sub_timers['advance_blocks']
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(20):
self.timer.show(height=height)
return undo
def _checksig(self, value, address):
try:
claim_dict = Claim.from_bytes(value)
cert_id = claim_dict.signing_channel_hash
if not self.should_validate_signatures:
return cert_id
if cert_id:
cert_claim = self.db.get_claim_info(cert_id)
if cert_claim:
certificate = Claim.from_bytes(cert_claim.value)
claim_dict.validate_signature(address, certificate)
return cert_id
except Exception:
pass
-21
View File
@@ -1,21 +0,0 @@
class FindShortestID:
__slots__ = 'short_id', 'new_id'
def __init__(self):
self.short_id = ''
self.new_id = None
def step(self, other_id, new_id):
self.new_id = new_id
for i in range(len(self.new_id)):
if other_id[i] != self.new_id[i]:
if i > len(self.short_id)-1:
self.short_id = self.new_id[:i+1]
break
def finalize(self):
return '#'+self.short_id
def register_canonical_functions(connection):
connection.create_aggregate("shortest_id", 2, FindShortestID)
File diff suppressed because it is too large. Load diff
-113
View File
@@ -1,113 +0,0 @@
from math import sqrt
TRENDING_WINDOW = 650 # number of blocks, ~24hr period
TRENDING_DATA_POINTS = 7 # WINDOW * DATA_POINTS = ~1 week worth of trending data
CREATE_TREND_TABLE = """
create table if not exists trend (
claim_hash bytes not null,
height integer not null,
amount integer not null,
primary key (claim_hash, height)
) without rowid;
"""
class ZScore:
__slots__ = 'count', 'total', 'power', 'last'
def __init__(self):
self.count = 0
self.total = 0
self.power = 0
self.last = None
def step(self, value):
if self.last is not None:
self.count += 1
self.total += self.last
self.power += self.last**2
self.last = value
@property
def mean(self):
return self.total / self.count
@property
def standard_deviation(self):
return sqrt((self.power / self.count) - self.mean**2)
def finalize(self):
if self.count == 0:
return self.last
return (self.last - self.mean) / (self.standard_deviation or 1)
def register_trending_functions(connection):
connection.create_aggregate("zscore", 1, ZScore)
def calculate_trending(db, height, is_first_sync, final_height):
# don't start tracking until we're at the end of initial sync
if is_first_sync and height < (final_height - (TRENDING_WINDOW*TRENDING_DATA_POINTS)):
return
if height % TRENDING_WINDOW != 0:
return
db.execute(f"""
DELETE FROM trend WHERE height < {height-(TRENDING_WINDOW*TRENDING_DATA_POINTS)}
""")
start = (height-TRENDING_WINDOW)+1
db.execute(f"""
INSERT INTO trend (claim_hash, height, amount)
SELECT claim_hash, {start}, COALESCE(
(SELECT SUM(amount) FROM support WHERE claim_hash=claim.claim_hash
AND height >= {start}), 0
) AS support_sum
FROM claim WHERE support_sum > 0
""")
zscore = ZScore()
for (global_sum,) in db.execute("SELECT AVG(amount) FROM trend GROUP BY height"):
zscore.step(global_sum)
global_mean, global_deviation = 0, 1
if zscore.count > 0:
global_mean = zscore.mean
global_deviation = zscore.standard_deviation
db.execute(f"""
UPDATE claim SET
trending_local = COALESCE((
SELECT zscore(amount) FROM trend
WHERE claim_hash=claim.claim_hash ORDER BY height DESC
), 0),
trending_global = COALESCE((
SELECT (amount - {global_mean}) / {global_deviation} FROM trend
WHERE claim_hash=claim.claim_hash AND height = {start}
), 0),
trending_group = 0,
trending_mixed = 0
""")
# trending_group and trending_mixed determine how trending will show in query results
# normally the SQL will be: "ORDER BY trending_group, trending_mixed"
# changing the trending_group will have significant impact on trending results
# changing the value used for trending_mixed will only impact trending within a trending_group
db.execute(f"""
UPDATE claim SET
trending_group = CASE
WHEN trending_local > 0 AND trending_global > 0 THEN 4
WHEN trending_local <= 0 AND trending_global > 0 THEN 3
WHEN trending_local > 0 AND trending_global <= 0 THEN 2
WHEN trending_local <= 0 AND trending_global <= 0 THEN 1
END,
trending_mixed = CASE
WHEN trending_local > 0 AND trending_global > 0 THEN trending_global
WHEN trending_local <= 0 AND trending_global > 0 THEN trending_local
WHEN trending_local > 0 AND trending_global <= 0 THEN trending_local
WHEN trending_local <= 0 AND trending_global <= 0 THEN trending_global
END
WHERE trending_local <> 0 OR trending_global <> 0
""")
-75
View File
@@ -1,75 +0,0 @@
import sys
import asyncio
from binascii import hexlify
from lbry.wallet.server.db import SQLDB
from lbry.wallet.server.coin import LBC
from lbry.wallet.server.daemon import Daemon
def hex_reverted(value: bytes) -> str:
return hexlify(value[::-1]).decode()
def match(name, what, value, expected):
if value != expected:
print(f'{name}: {what} mismatch, {value} is not {expected}')
return value == expected
def checkrecord(record, expected_winner, expected_claim):
assert record['is_controlling'] == record['claim_hash'], dict(record)
name = record['normalized']
claim_id = hex_reverted(record['claim_hash'])
takover = record['activation_height']
if not expected_winner:
print(f"{name} not on lbrycrd. We have {claim_id} at {takover} takeover height.")
return
if not match(name, 'claim id', claim_id, expected_winner['claimId']):
print(f"-- {name} has the wrong winner")
if not expected_claim:
print(f'{name}: {claim_id} not found, we possibly have an abandoned claim as winner')
return
match(name, 'height', record['height'], expected_claim['height'])
match(name, 'activation height', takover, expected_claim['valid at height'])
match(name, 'name', record['normalized'], expected_claim['normalized_name'])
match(name, 'amount', record['amount'], expected_claim['amount'])
match(name, 'effective amount', record['effective_amount'], expected_claim['effective amount'])
match(name, 'txid', hex_reverted(record['txo_hash'][:-4]), expected_claim['txid'])
match(name, 'nout', int.from_bytes(record['txo_hash'][-4:], 'little', signed=False), expected_claim['n'])
async def checkcontrolling(daemon: Daemon, db: SQLDB):
records, claim_ids, names, futs = [], [], [], []
for record in db.get_claims('claimtrie.claim_hash as is_controlling, claim.*', is_controlling=True):
records.append(record)
claim_id = hex_reverted(record['claim_hash'])
claim_ids.append((claim_id,))
names.append((record['normalized'],))
if len(names) > 50000:
futs.append(daemon._send_vector('getvalueforname', names[:]))
futs.append(daemon._send_vector('getclaimbyid', claim_ids[:]))
names.clear()
claim_ids.clear()
if names:
futs.append(daemon._send_vector('getvalueforname', names[:]))
futs.append(daemon._send_vector('getclaimbyid', claim_ids[:]))
names.clear()
claim_ids.clear()
while futs:
winners, claims = futs.pop(0), futs.pop(0)
for winner, claim in zip(await winners, await claims):
checkrecord(records.pop(0), winner, claim)
if __name__ == '__main__':
if not len(sys.argv) == 3:
print("usage: <db_file_path> <lbrycrd_url>")
sys.exit(1)
db_path, lbrycrd_url = sys.argv[1:] # pylint: disable=W0632
daemon = Daemon(LBC(), url=lbrycrd_url)
db = SQLDB(None, db_path)
db.open()
asyncio.get_event_loop().run_until_complete(checkcontrolling(daemon, db))
-17
View File
@@ -1,17 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/torba" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/torba/tests" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lbry" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lbry/tests" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lbry/scripts" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="Python 3.7 (lbry)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
-8
View File
@@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/lbry-sdk.iml" filepath="$PROJECT_DIR$/.idea/lbry-sdk.iml" />
</modules>
</component>
</project>
-6
View File
@@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
-36
View File
@@ -1,36 +0,0 @@
"""Set the build version to be 'qa', 'rc', 'release'"""
import sys
import os
import re
import logging
log = logging.getLogger()
log.addHandler(logging.StreamHandler())
log.setLevel(logging.DEBUG)
def get_build_type(travis_tag=None):
if not travis_tag:
return "qa"
log.debug("getting build type for tag: \"%s\"", travis_tag)
if re.match('v\d+\.\d+\.\d+rc\d+$', travis_tag):
return 'rc'
elif re.match('v\d+\.\d+\.\d+$', travis_tag):
return 'release'
return 'qa'
def main():
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
build_type_path = os.path.join(root_dir, 'lbry', 'build_type.py')
log.debug("configuring build type file: %s", build_type_path)
travis_commit = os.environ['TRAVIS_COMMIT'][:6]
build_type = get_build_type(os.environ.get('TRAVIS_TAG', None))
log.debug("setting build type=%s, build commit=%s", build_type, travis_commit)
with open(build_type_path, 'w') as f:
f.write("BUILD = \"{}\"\nBUILD_COMMIT = \"{}\"\n".format(build_type, travis_commit))
if __name__ == '__main__':
sys.exit(main())
@@ -1,175 +0,0 @@
from math import ceil
from aiohttp import web
from lbry.testcase import CommandTestCase
class MockedCommentServer:
ERRORS = {
'INVALID_PARAMS': {'code': -32602, 'message': 'Invalid parameters'},
'INTERNAL': {'code': -32603, 'message': 'An internal error'},
'UNKNOWN': {'code': -1, 'message': 'An unknown or very miscellaneous error'},
'INVALID_METHOD': {'code': -32604, 'message': 'The Requested method does not exist'}
}
def __init__(self, port=2903):
self.port = port
self.app = web.Application(debug=True)
self.app.add_routes([web.post('/api', self.api)])
self.runner = None
self.server = None
self.comments = []
self.comment_id = 0
def create_comment(self, **comment):
self.comment_id += 1
comment['comment_id'] = self.comment_id
if 'channel_id' in comment:
comment['channel_url'] = 'lbry://' + comment['channel_name'] + '#' + comment['channel_id']
self.comments.append(comment)
return comment
def get_claim_comments(self, page=1, page_size=50, **kwargs):
return {
'page': page,
'page_size': page_size,
'total_pages': ceil(len(self.comments)/page_size),
'total_items': len(self.comments),
'items': (self.comments[::-1])[(page - 1) * page_size: page * page_size]
}
methods = {
'get_claim_comments': get_claim_comments,
'create_comment': create_comment,
}
def process_json(self, body) -> dict:
response = {'jsonrpc': '2.0', 'id': body['id']}
if body['method'] in self.methods:
params = body.get('params', {})
result = self.methods[body['method']](self, **params)
response['result'] = result
else:
response['error'] = self.ERRORS['INVALID_METHOD']
return response
async def start(self):
self.runner = web.AppRunner(self.app)
await self.runner.setup()
self.server = web.TCPSite(self.runner, 'localhost', self.port)
await self.server.start()
async def stop(self):
await self.runner.shutdown()
await self.runner.cleanup()
async def api(self, request):
body = await request.json()
if type(body) is list or type(body) is dict:
if type(body) is list:
response = [self.process_json(part) for part in body]
else:
response = self.process_json(body)
return web.json_response(response)
else:
raise TypeError('invalid type passed')
class CommentCommands(CommandTestCase):
async def asyncSetUp(self):
await super().asyncSetUp()
self.daemon.conf.comment_server = 'http://localhost:2903/api'
self.comment_server = MockedCommentServer(2903)
await self.comment_server.start()
self.addCleanup(self.comment_server.stop)
async def test01_comment_create(self):
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
stream = (await self.stream_create())['outputs'][0]
self.assertEqual(0, len((await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']))
comment = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
channel_id=channel['claim_id'],
comment="It's 5 O'Clock Somewhere"
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(1, len(comments))
self.assertEqual(comment['comment_id'], comments[0]['comment_id'])
self.assertEqual(stream['claim_id'], comments[0]['claim_id'])
channel2 = (await self.channel_create('@BuffettJimmy'))['outputs'][0]
await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
channel_name=channel2['name'],
comment='Let\'s all go to Margaritaville',
parent_id=comments[0]['comment_id']
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(2, len(comments))
self.assertEqual(comments[0]['channel_id'], channel2['claim_id'])
self.assertEqual(comments[0]['parent_id'], comments[1]['comment_id'])
comment = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
comment='Anonymous comment'
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(comment['comment_id'], comments[0]['comment_id'])
async def test02_unsigned_comment_list(self):
stream = (await self.stream_create())['outputs'][0]
comments = []
for i in range(28):
comment = await self.daemon.jsonrpc_comment_create(
comment=f'{i}',
claim_id=stream['claim_id'],
)
self.assertIn('comment_id', comment)
comments.append(comment)
comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id']
)
self.assertIs(comment_list['page_size'], 50)
self.assertIs(comment_list['page'], 1)
self.assertIs(comment_list['total_items'], 28)
for comment in comment_list['items']:
self.assertEqual(comment['comment'], comments.pop()['comment'])
signed_comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id'],
is_channel_signature_valid=True
)
self.assertIs(len(signed_comment_list['items']), 0)
async def test03_signed_comments_list(self):
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
stream = (await self.stream_create())['outputs'][0]
comments = []
for i in range(28):
comment = await self.daemon.jsonrpc_comment_create(
comment=f'{i}',
claim_id=stream['claim_id'],
channel_id=channel['claim_id'],
)
self.assertIn('comment_id', comment)
comments.append(comment)
comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id']
)
self.assertIs(comment_list['page_size'], 50)
self.assertIs(comment_list['page'], 1)
self.assertIs(comment_list['total_items'], 28)
for comment in comment_list['items']:
self.assertEqual(comment['comment'], comments.pop()['comment'])
signed_comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id'],
is_channel_signature_valid=True
)
self.assertIs(len(signed_comment_list['items']), 28)
-115
View File
@@ -1,115 +0,0 @@
import asyncio
from binascii import hexlify
from lbry.dht import constants
from lbry.dht.node import Node
from lbry.dht.peer import PeerManager, KademliaPeer
from torba.testcase import AsyncioTestCase
class DHTIntegrationTest(AsyncioTestCase):
async def asyncSetUp(self):
import logging
logging.getLogger('asyncio').setLevel(logging.ERROR)
logging.getLogger('lbry.dht').setLevel(logging.WARN)
self.nodes = []
self.known_node_addresses = []
async def setup_network(self, size: int, start_port=40000, seed_nodes=1):
for i in range(size):
node_port = start_port + i
node = Node(self.loop, PeerManager(self.loop), node_id=constants.generate_id(i),
udp_port=node_port, internal_udp_port=node_port,
peer_port=3333, external_ip='127.0.0.1')
self.nodes.append(node)
self.known_node_addresses.append(('127.0.0.1', node_port))
await node.start_listening('127.0.0.1')
self.addCleanup(node.stop)
for node in self.nodes:
node.protocol.rpc_timeout = .5
node.protocol.ping_queue._default_delay = .5
node.start('127.0.0.1', self.known_node_addresses[:seed_nodes])
await asyncio.gather(*[node.joined.wait() for node in self.nodes])
async def test_replace_bad_nodes(self):
await self.setup_network(20)
self.assertEquals(len(self.nodes), 20)
node = self.nodes[0]
bad_peers = []
for candidate in self.nodes[1:10]:
address, port, node_id = candidate.protocol.external_ip, candidate.protocol.udp_port, candidate.protocol.node_id
peer = KademliaPeer(self.loop, address, node_id, port)
bad_peers.append(peer)
node.protocol.add_peer(peer)
candidate.stop()
await asyncio.sleep(.3) # let pending events settle
for bad_peer in bad_peers:
self.assertIn(bad_peer, node.protocol.routing_table.get_peers())
await node.refresh_node(True)
await asyncio.sleep(.3) # let pending events settle
good_nodes = {good_node.protocol.node_id for good_node in self.nodes[10:]}
for peer in node.protocol.routing_table.get_peers():
self.assertIn(peer.node_id, good_nodes)
async def test_re_join(self):
await self.setup_network(20, seed_nodes=10)
node = self.nodes[-1]
self.assertTrue(node.joined.is_set())
self.assertTrue(node.protocol.routing_table.get_peers())
for network_node in self.nodes[:-1]:
network_node.stop()
await node.refresh_node(True)
await asyncio.sleep(.3) # let pending events settle
self.assertFalse(node.protocol.routing_table.get_peers())
for network_node in self.nodes[:-1]:
network_node.start('127.0.0.1', self.known_node_addresses)
self.assertFalse(node.protocol.routing_table.get_peers())
timeout = 20
while not node.protocol.routing_table.get_peers():
await asyncio.sleep(.1)
timeout -= 1
if not timeout:
self.fail("node didnt join back after 2 seconds")
async def test_announce_no_peers(self):
await self.setup_network(1)
node = self.nodes[0]
blob_hash = hexlify(constants.generate_id(1337)).decode()
peers = await node.announce_blob(blob_hash)
self.assertEqual(len(peers), 0)
async def test_get_token_on_announce(self):
await self.setup_network(2, seed_nodes=2)
node1, node2 = self.nodes
node1.protocol.peer_manager.clear_token(node2.protocol.node_id)
blob_hash = hexlify(constants.generate_id(1337)).decode()
node_ids = await node1.announce_blob(blob_hash)
self.assertIn(node2.protocol.node_id, node_ids)
node2.protocol.node_rpc.refresh_token()
node_ids = await node1.announce_blob(blob_hash)
self.assertIn(node2.protocol.node_id, node_ids)
node2.protocol.node_rpc.refresh_token()
node_ids = await node1.announce_blob(blob_hash)
self.assertIn(node2.protocol.node_id, node_ids)
async def test_peer_search_removes_bad_peers(self):
# that's an edge case discovered by Tom, but an important one
# imagine that you only got bad peers and refresh will happen in one hour
# instead of failing for one hour we should be able to recover by scheduling pings to bad peers we find
await self.setup_network(2, seed_nodes=2)
node1, node2 = self.nodes
node2.stop()
# forcefully make it a bad peer but dont remove it from routing table
address, port, node_id = node2.protocol.external_ip, node2.protocol.udp_port, node2.protocol.node_id
peer = KademliaPeer(self.loop, address, node_id, port)
self.assertTrue(node1.protocol.peer_manager.peer_is_good(peer))
node1.protocol.peer_manager.report_failure(node2.protocol.external_ip, node2.protocol.udp_port)
node1.protocol.peer_manager.report_failure(node2.protocol.external_ip, node2.protocol.udp_port)
self.assertFalse(node1.protocol.peer_manager.peer_is_good(peer))
# now a search happens, which removes bad peers while contacting them
self.assertTrue(node1.protocol.routing_table.get_peers())
await node1.peer_search(node2.protocol.node_id)
await asyncio.sleep(.3) # let pending events settle
self.assertFalse(node1.protocol.routing_table.get_peers())
@@ -1,391 +0,0 @@
import asyncio
import json
import hashlib
from unittest import skip
from binascii import hexlify, unhexlify
from lbry.testcase import CommandTestCase
from lbry.wallet.transaction import Transaction, Output
from lbry.schema.compat import OldClaimMessage
from torba.client.hash import sha256, Base58
class BaseResolveTestCase(CommandTestCase):
async def assertResolvesToClaimId(self, name, claim_id):
other = (await self.resolve(name))[name]
if claim_id is None:
self.assertIn('error', other)
else:
self.assertEqual(claim_id, other['claim_id'])
class ResolveCommand(BaseResolveTestCase):
async def test_resolve_response(self):
channel_id = self.get_claim_id(
await self.channel_create('@abc', '0.01')
)
# resolving a channel @abc
response = await self.resolve('lbry://@abc')
self.assertSetEqual({'lbry://@abc'}, set(response))
self.assertEqual(response['lbry://@abc']['name'], '@abc')
self.assertEqual(response['lbry://@abc']['value_type'], 'channel')
self.assertEqual(response['lbry://@abc']['meta']['claims_in_channel'], 0)
await self.stream_create('foo', '0.01', channel_id=channel_id)
await self.stream_create('foo2', '0.01', channel_id=channel_id)
# resolving a channel @abc with some claims in it
response['lbry://@abc']['confirmations'] += 2
response['lbry://@abc']['meta']['claims_in_channel'] = 2
self.assertEqual(response, await self.resolve('lbry://@abc'))
# resolving claim foo within channel @abc
response = await self.resolve('lbry://@abc/foo')
self.assertSetEqual({'lbry://@abc/foo'}, set(response))
claim = response['lbry://@abc/foo']
self.assertEqual(claim['name'], 'foo')
self.assertEqual(claim['value_type'], 'stream')
self.assertEqual(claim['signing_channel']['name'], '@abc')
self.assertTrue(claim['is_channel_signature_valid'])
self.assertEqual(
claim['timestamp'],
self.ledger.headers[claim['height']]['timestamp']
)
self.assertEqual(
claim['signing_channel']['timestamp'],
self.ledger.headers[claim['signing_channel']['height']]['timestamp']
)
# resolving claim foo by itself
self.assertEqual(claim, (await self.resolve('lbry://foo'))['lbry://foo'])
# resolving from the given permanent url
permanent_url = response['lbry://@abc/foo']['permanent_url']
self.assertEqual(claim, (await self.resolve(permanent_url))[permanent_url])
# resolving multiple at once
response = await self.resolve(['lbry://foo', 'lbry://foo2'])
self.assertSetEqual({'lbry://foo', 'lbry://foo2'}, set(response))
claim = response['lbry://foo2']
self.assertEqual(claim['name'], 'foo2')
self.assertEqual(claim['value_type'], 'stream')
self.assertEqual(claim['signing_channel']['name'], '@abc')
self.assertTrue(claim['is_channel_signature_valid'])
# resolve has correct confirmations
tx_details = await self.blockchain.get_raw_transaction(claim['txid'])
self.assertEqual(claim['confirmations'], json.loads(tx_details)['confirmations'])
# resolve handles invalid data
await self.blockchain_claim_name("gibberish", hexlify(b"{'invalid':'json'}").decode(), "0.1")
await self.generate(1)
response = await self.resolve("lbry://gibberish")
self.assertSetEqual({'lbry://gibberish'}, set(response))
claim = response['lbry://gibberish']
self.assertEqual(claim['name'], 'gibberish')
self.assertNotIn('value', claim)
async def test_winning_by_effective_amount(self):
# first one remains winner unless something else changes
claim_id1 = self.get_claim_id(
await self.channel_create('@foo', allow_duplicate_name=True))
await self.assertResolvesToClaimId('@foo', claim_id1)
claim_id2 = self.get_claim_id(
await self.channel_create('@foo', allow_duplicate_name=True))
await self.assertResolvesToClaimId('@foo', claim_id1)
claim_id3 = self.get_claim_id(
await self.channel_create('@foo', allow_duplicate_name=True))
await self.assertResolvesToClaimId('@foo', claim_id1)
# supports change the winner
await self.support_create(claim_id3, '0.09')
await self.assertResolvesToClaimId('@foo', claim_id3)
await self.support_create(claim_id2, '0.19')
await self.assertResolvesToClaimId('@foo', claim_id2)
await self.support_create(claim_id1, '0.29')
await self.assertResolvesToClaimId('@foo', claim_id1)
async def test_advanced_resolve(self):
claim_id1 = self.get_claim_id(
await self.stream_create('foo', '0.7', allow_duplicate_name=True))
claim_id2 = self.get_claim_id(
await self.stream_create('foo', '0.8', allow_duplicate_name=True))
claim_id3 = self.get_claim_id(
await self.stream_create('foo', '0.9', allow_duplicate_name=True))
# plain winning claim
await self.assertResolvesToClaimId('foo', claim_id3)
# sequence resolution
await self.assertResolvesToClaimId('foo:1', claim_id1)
await self.assertResolvesToClaimId('foo:2', claim_id2)
await self.assertResolvesToClaimId('foo:3', claim_id3)
await self.assertResolvesToClaimId('foo:4', None)
# amount order resolution
await self.assertResolvesToClaimId('foo$1', claim_id3)
await self.assertResolvesToClaimId('foo$2', claim_id2)
await self.assertResolvesToClaimId('foo$3', claim_id1)
await self.assertResolvesToClaimId('foo$4', None)
async def test_partial_claim_id_resolve(self):
# add some noise
await self.channel_create('@abc', '0.1', allow_duplicate_name=True)
await self.channel_create('@abc', '0.2', allow_duplicate_name=True)
await self.channel_create('@abc', '1.0', allow_duplicate_name=True)
channel_id = self.get_claim_id(
await self.channel_create('@abc', '1.1', allow_duplicate_name=True))
await self.assertResolvesToClaimId(f'@abc', channel_id)
await self.assertResolvesToClaimId(f'@abc#{channel_id[:10]}', channel_id)
await self.assertResolvesToClaimId(f'@abc#{channel_id}', channel_id)
channel = (await self.claim_search(claim_id=channel_id))[0]
await self.assertResolvesToClaimId(channel['short_url'], channel_id)
await self.assertResolvesToClaimId(channel['canonical_url'], channel_id)
await self.assertResolvesToClaimId(channel['permanent_url'], channel_id)
# add some noise
await self.stream_create('foo', '0.1', allow_duplicate_name=True, channel_id=channel['claim_id'])
await self.stream_create('foo', '0.2', allow_duplicate_name=True, channel_id=channel['claim_id'])
await self.stream_create('foo', '0.3', allow_duplicate_name=True, channel_id=channel['claim_id'])
claim_id1 = self.get_claim_id(
await self.stream_create('foo', '0.7', allow_duplicate_name=True, channel_id=channel['claim_id']))
claim1 = (await self.claim_search(claim_id=claim_id1))[0]
await self.assertResolvesToClaimId('foo', claim_id1)
await self.assertResolvesToClaimId('@abc/foo', claim_id1)
await self.assertResolvesToClaimId(claim1['short_url'], claim_id1)
await self.assertResolvesToClaimId(claim1['canonical_url'], claim_id1)
await self.assertResolvesToClaimId(claim1['permanent_url'], claim_id1)
claim_id2 = self.get_claim_id(
await self.stream_create('foo', '0.8', allow_duplicate_name=True, channel_id=channel['claim_id']))
claim2 = (await self.claim_search(claim_id=claim_id2))[0]
await self.assertResolvesToClaimId('foo', claim_id2)
await self.assertResolvesToClaimId('@abc/foo', claim_id2)
await self.assertResolvesToClaimId(claim2['short_url'], claim_id2)
await self.assertResolvesToClaimId(claim2['canonical_url'], claim_id2)
await self.assertResolvesToClaimId(claim2['permanent_url'], claim_id2)
async def test_abandoned_channel_with_signed_claims(self):
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
orphan_claim = await self.stream_create('on-channel-claim', '0.0001', channel_id=channel['claim_id'])
abandoned_channel_id = channel['claim_id']
await self.channel_abandon(txid=channel['txid'], nout=0)
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
orphan_claim_id = self.get_claim_id(orphan_claim)
# Original channel doesnt exists anymore, so the signature is invalid. For invalid signatures, resolution is
# only possible outside a channel
response = await self.resolve('lbry://@abc/on-channel-claim')
self.assertEqual(response, {
'lbry://@abc/on-channel-claim': {'error': 'lbry://@abc/on-channel-claim did not resolve to a claim'}
})
response = (await self.resolve('lbry://on-channel-claim'))['lbry://on-channel-claim']
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
direct_uri = 'lbry://on-channel-claim#' + orphan_claim_id
response = (await self.resolve(direct_uri))[direct_uri]
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
await self.stream_abandon(claim_id=orphan_claim_id)
uri = 'lbry://@abc/on-channel-claim'
# now, claim something on this channel (it will update the invalid claim, but we save and forcefully restore)
valid_claim = await self.stream_create('on-channel-claim', '0.00000001', channel_id=channel['claim_id'])
# resolves normally
response = await self.resolve(uri)
self.assertTrue(response[uri]['is_channel_signature_valid'])
# ooops! claimed a valid conflict! (this happens on the wild, mostly by accident or race condition)
await self.stream_create(
'on-channel-claim', '0.00000001', channel_id=channel['claim_id'], allow_duplicate_name=True
)
# it still resolves! but to the older claim
response = await self.resolve(uri)
self.assertTrue(response[uri]['is_channel_signature_valid'])
self.assertEqual(response[uri]['txid'], valid_claim['txid'])
claims = await self.claim_search(name='on-channel-claim')
self.assertEqual(2, len(claims))
self.assertEqual(
{channel['claim_id']}, {claim['signing_channel']['claim_id'] for claim in claims}
)
async def test_normalization_resolution(self):
one = 'ΣίσυφοςfiÆ'
two = 'ΣΊΣΥΦΟσFIæ'
_ = await self.stream_create(one, '0.1')
c = await self.stream_create(two, '0.2')
winner_id = self.get_claim_id(c)
r1 = await self.resolve(f'lbry://{one}')
r2 = await self.resolve(f'lbry://{two}')
self.assertEqual(winner_id, r1[f'lbry://{one}']['claim_id'])
self.assertEqual(winner_id, r2[f'lbry://{two}']['claim_id'])
async def test_resolve_old_claim(self):
channel = await self.daemon.jsonrpc_channel_create('@olds', '1.0')
await self.confirm_tx(channel.id)
address = channel.outputs[0].get_address(self.account.ledger)
claim = generate_signed_legacy(address, channel.outputs[0])
tx = await Transaction.claim_create('example', claim.SerializeToString(), 1, address, [self.account], self.account)
await tx.sign([self.account])
await self.broadcast(tx)
await self.confirm_tx(tx.id)
response = await self.resolve('@olds/example')
self.assertTrue(response['@olds/example']['is_channel_signature_valid'])
claim.publisherSignature.signature = bytes(reversed(claim.publisherSignature.signature))
tx = await Transaction.claim_create(
'bad_example', claim.SerializeToString(), 1, address, [self.account], self.account
)
await tx.sign([self.account])
await self.broadcast(tx)
await self.confirm_tx(tx.id)
response = await self.resolve('bad_example')
self.assertFalse(response['bad_example']['is_channel_signature_valid'])
response = await self.resolve('@olds/bad_example')
self.assertEqual(response, {
'@olds/bad_example': {'error': '@olds/bad_example did not resolve to a claim'}
})
async def _test_resolve_abc_foo(self):
response = await self.resolve('lbry://@abc/foo')
claim = response['lbry://@abc/foo']
self.assertIn('signing_channel', claim)
self.assertEqual(claim['name'], 'foo')
self.assertEqual(claim['signing_channel']['name'], '@abc')
self.assertEqual(claim['meta']['claims_in_channel'], 0)
self.assertEqual(
claim['timestamp'],
self.ledger.headers[claim['height']]['timestamp']
)
self.assertEqual(
claim['signing_channel']['timestamp'],
self.ledger.headers[claim['signing_channel']['height']]['timestamp']
)
@skip('this test does not work with new resolve')
async def test_resolve_lru_cache_doesnt_persist_errors(self):
original_get_transaction = self.daemon.wallet_manager.ledger.network.get_transaction
async def timeout_get_transaction(txid):
fut = self.loop.create_future()
def delayed_raise_cancelled_error():
fut.set_exception(asyncio.CancelledError())
self.loop.call_soon(delayed_raise_cancelled_error)
return await fut
tx = await self.channel_create('@abc', '0.01')
channel_id = self.get_claim_id(tx)
await self.stream_create('foo', '0.01', channel_id=channel_id)
# raise a cancelled error from get_transaction
self.daemon.wallet_manager.ledger.network.get_transaction = timeout_get_transaction
with self.assertRaises(KeyError):
await self._test_resolve_abc_foo()
# restore the real get_transaction that doesn't cancel, it should be called and the result cached
self.daemon.wallet_manager.ledger.network.get_transaction = original_get_transaction
await self._test_resolve_abc_foo()
called_again = asyncio.Event(loop=self.loop)
def check_result_cached(txid):
called_again.set()
return original_get_transaction(txid)
# check that the result was cached
self.daemon.wallet_manager.ledger.network.get_transaction = check_result_cached
await self._test_resolve_abc_foo()
self.assertFalse(called_again.is_set())
class ResolveAfterReorg(BaseResolveTestCase):
async def reorg(self, start):
blocks = self.ledger.headers.height - start
self.blockchain._block_expected = start - 1
# go back to start
await self.blockchain.invalidate_block(self.ledger.headers.hash(start).decode())
# go to previous + 1
await self.generate(blocks + 2)
async def test_reorg(self):
self.assertEqual(self.ledger.headers.height, 206)
channel_name = '@abc'
channel_id = self.get_claim_id(
await self.channel_create(channel_name, '0.01')
)
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
await self.reorg(206)
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
stream_name = 'foo'
stream_id = self.get_claim_id(
await self.stream_create(stream_name, '0.01', channel_id=channel_id)
)
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
await self.reorg(206)
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
await self.support_create(stream_id, '0.01')
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
await self.reorg(206)
self.assertNotIn('error', (await self.resolve(stream_name))[stream_name])
await self.stream_abandon(stream_id)
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
await self.reorg(206)
self.assertNotIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
await self.channel_abandon(channel_id)
self.assertIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
await self.reorg(206)
self.assertIn('error', (await self.resolve(channel_name))[channel_name])
self.assertIn('error', (await self.resolve(stream_name))[stream_name])
def generate_signed_legacy(address: bytes, output: Output):
decoded_address = Base58.decode(address)
claim = OldClaimMessage()
claim.ParseFromString(unhexlify(
'080110011aee04080112a604080410011a2b4865726520617265203520526561736f6e73204920e29da4e'
'fb88f204e657874636c6f7564207c20544c4722920346696e64206f7574206d6f72652061626f7574204e'
'657874636c6f75643a2068747470733a2f2f6e657874636c6f75642e636f6d2f0a0a596f752063616e206'
'6696e64206d65206f6e20746865736520736f6369616c733a0a202a20466f72756d733a2068747470733a'
'2f2f666f72756d2e6865617679656c656d656e742e696f2f0a202a20506f64636173743a2068747470733'
'a2f2f6f6666746f706963616c2e6e65740a202a2050617472656f6e3a2068747470733a2f2f7061747265'
'6f6e2e636f6d2f7468656c696e757867616d65720a202a204d657263683a2068747470733a2f2f7465657'
'37072696e672e636f6d2f73746f7265732f6f6666696369616c2d6c696e75782d67616d65720a202a2054'
'77697463683a2068747470733a2f2f7477697463682e74762f786f6e64616b0a202a20547769747465723'
'a2068747470733a2f2f747769747465722e636f6d2f7468656c696e757867616d65720a0a2e2e2e0a6874'
'7470733a2f2f7777772e796f75747562652e636f6d2f77617463683f763d4672546442434f535f66632a0'
'f546865204c696e75782047616d6572321c436f7079726967687465642028636f6e746163742061757468'
'6f722938004a2968747470733a2f2f6265726b2e6e696e6a612f7468756d626e61696c732f46725464424'
'34f535f666352005a001a41080110011a30040e8ac6e89c061f982528c23ad33829fd7146435bf7a4cc22'
'f0bff70c4fe0b91fd36da9a375e3e1c171db825bf5d1f32209766964656f2f6d70342a5c080110031a406'
'2b2dd4c45e364030fbfad1a6fefff695ebf20ea33a5381b947753e2a0ca359989a5cc7d15e5392a0d354c'
'0b68498382b2701b22c03beb8dcb91089031b871e72214feb61536c007cdf4faeeaab4876cb397feaf6b51'
))
claim.ClearField("publisherSignature")
digest = sha256(b''.join([
decoded_address,
claim.SerializeToString(),
output.claim_hash[::-1]
]))
signature = output.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
claim.publisherSignature.version = 1
claim.publisherSignature.signatureType = 1
claim.publisherSignature.signature = signature
claim.publisherSignature.certificateId = output.claim_hash[::-1]
return claim
@@ -1,33 +0,0 @@
from torba.testcase import AsyncioTestCase
from lbry.extras.daemon.comment_client import sign_comment
from lbry.extras.daemon.comment_client import is_comment_signed_by_channel
from tests.unit.wallet.test_schema_signing import get_stream, get_channel
class TestSigningComments(AsyncioTestCase):
@staticmethod
def create_claim_comment_body(comment, claim, channel):
return {
'claim_id': claim.claim_id,
'channel_name': channel.claim_name,
'channel_id': channel.claim_id,
'comment': comment
}
def test01_successful_create_sign_and_validate_comment(self):
channel = get_channel('@BusterBluth')
stream = get_stream('pop secret')
comment = self.create_claim_comment_body('Cool stream', stream, channel)
sign_comment(comment, channel)
self.assertTrue(is_comment_signed_by_channel(comment, channel))
def test02_fail_to_validate_spoofed_channel(self):
pdiddy = get_channel('@PDitty')
channel2 = get_channel('@TomHaverford')
stream = get_stream()
comment = self.create_claim_comment_body('Woahh This is Sick!! Shout out 2 my boy Tommy H', stream, pdiddy)
sign_comment(comment, channel2)
self.assertFalse(is_comment_signed_by_channel(comment, pdiddy))
-19
View File
@@ -1,19 +0,0 @@
import unittest
from lbry.schema.tags import normalize_tag, clean_tags
class TestTagNormalization(unittest.TestCase):
def assertNormalizedTag(self, clean, dirty):
self.assertEqual(clean, normalize_tag(dirty))
def test_normalize_tag(self):
tag = self.assertNormalizedTag
tag('', ' \t #!~')
tag('tag', 'Tag')
tag('t ag', '\tT \nAG ')
tag('tag hash', '#tag~#hash!')
def test_clean_tags(self):
self.assertEqual(['tag'], clean_tags([' \t #!~', '!taG', '\t']))
-88
View File
@@ -1,88 +0,0 @@
import unittest
from lbry.schema.url import URL
claim_id = "63f2da17b0d90042c559cc73b6b17f853945c43e"
class TestURLParsing(unittest.TestCase):
segments = 'stream', 'channel'
fields = 'name', 'claim_id', 'sequence', 'amount_order'
def _assert_url(self, url_string, **kwargs):
url = URL.parse(url_string)
if url_string.startswith('lbry://'):
self.assertEqual(url_string, str(url))
else:
self.assertEqual(f'lbry://{url_string}', str(url))
present = {}
for key in kwargs:
for segment_name in self.segments:
if key.startswith(segment_name):
present[segment_name] = True
break
for segment_name in self.segments:
segment = getattr(url, segment_name)
if segment_name not in present:
self.assertIsNone(segment)
else:
for field in self.fields:
self.assertEqual(
getattr(segment, field),
kwargs.get(f'{segment_name}_{field}', None)
)
def _fail_url(self, url):
with self.assertRaisesRegex(ValueError, 'Invalid LBRY URL'):
URL.parse(url)
def test_parser_valid_urls(self):
url = self._assert_url
# stream
url('test', stream_name='test')
url('test:1', stream_name='test', stream_sequence='1')
url('test$1', stream_name='test', stream_amount_order='1')
url(f'test#{claim_id}', stream_name='test', stream_claim_id=claim_id)
# channel
url('@test', channel_name='@test')
url('@test:1', channel_name='@test', channel_sequence='1')
url('@test$1', channel_name='@test', channel_amount_order='1')
url(f'@test#{claim_id}', channel_name='@test', channel_claim_id=claim_id)
# channel/stream
url('lbry://@test/stuff', channel_name='@test', stream_name='stuff')
url('lbry://@test:1/stuff', channel_name='@test', channel_sequence='1', stream_name='stuff')
url('lbry://@test$1/stuff', channel_name='@test', channel_amount_order='1', stream_name='stuff')
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
def test_parser_invalid_urls(self):
fail = self._fail_url
fail("lbry://")
fail("lbry://test:3$1")
fail("lbry://test$1:1")
fail("lbry://test#x")
fail("lbry://test#x/page")
fail("lbry://test$")
fail("lbry://test#")
fail("lbry://test:")
fail("lbry://test$x")
fail("lbry://test:x")
fail("lbry://@test@")
fail("lbry://@test:")
fail("lbry://test@")
fail("lbry://tes@t")
fail(f"lbry://test:1#{claim_id}")
fail("lbry://test:0")
fail("lbry://test$0")
fail("lbry://test/path")
fail("lbry://@test1:1ab/fakepath")
fail("lbry://test:1:1:1")
fail("whatever/lbry://test")
fail("lbry://lbry://test")
fail("lbry://@/what")
fail("lbry://abc:0x123")
fail("lbry://abc:0x123/page")
fail("lbry://@test1#ABCDEF/fakepath")
fail("test:0001")
fail("lbry://@test1$1/fakepath?arg1&arg2&arg3")
View File
Whitespace-only changes.
Whitespace-only changes.
-444
View File
@@ -1,444 +0,0 @@
import unittest
import ecdsa
import hashlib
from binascii import hexlify
from torba.client.constants import COIN, NULL_HASH32
from lbry.schema.claim import Claim
from lbry.wallet.server.db import SQLDB
from lbry.wallet.server.coin import LBCRegTest
from lbry.wallet.server.trending import TRENDING_WINDOW
from lbry.wallet.server.canonical import FindShortestID
from lbry.wallet.server.block_processor import Timer
from lbry.wallet.transaction import Transaction, Input, Output
def get_output(amount=COIN, pubkey_hash=NULL_HASH32):
return Transaction() \
.add_outputs([Output.pay_pubkey_hash(amount, pubkey_hash)]) \
.outputs[0]
def get_input():
return Input.spend(get_output())
def get_tx():
return Transaction().add_inputs([get_input()])
class OldWalletServerTransaction:
def __init__(self, tx):
self.tx = tx
def serialize(self):
return self.tx.raw
class TestSQLDB(unittest.TestCase):
def setUp(self):
self.first_sync = False
self.daemon_height = 1
self.coin = LBCRegTest()
self.sql = SQLDB(self, ':memory:')
self.timer = Timer('BlockProcessor')
self.sql.open()
self._current_height = 0
self._txos = {}
def _make_tx(self, output, txi=None):
tx = get_tx().add_outputs([output])
if txi is not None:
tx.add_inputs([txi])
self._txos[output.ref.hash] = output
return OldWalletServerTransaction(tx), tx.hash
def _set_channel_key(self, channel, key):
private_key = ecdsa.SigningKey.from_string(key*32, curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
channel.private_key = private_key
channel.claim.channel.public_key_bytes = private_key.get_verifying_key().to_der()
channel.script.generate()
def get_channel(self, title, amount, name='@foo', key=b'a'):
claim = Claim()
claim.channel.title = title
channel = Output.pay_claim_name_pubkey_hash(amount, name, claim, b'abc')
self._set_channel_key(channel, key)
return self._make_tx(channel)
def get_channel_update(self, channel, amount, key=b'a'):
self._set_channel_key(channel, key)
return self._make_tx(
Output.pay_update_claim_pubkey_hash(
amount, channel.claim_name, channel.claim_id, channel.claim, b'abc'
),
Input.spend(channel)
)
def get_stream(self, title, amount, name='foo', channel=None):
claim = Claim()
claim.stream.title = title
result = self._make_tx(Output.pay_claim_name_pubkey_hash(amount, name, claim, b'abc'))
if channel:
result[0].tx.outputs[0].sign(channel)
result[0].tx._reset()
return result
def get_stream_update(self, tx, amount):
claim = Transaction(tx[0].serialize()).outputs[0]
return self._make_tx(
Output.pay_update_claim_pubkey_hash(
amount, claim.claim_name, claim.claim_id, claim.claim, b'abc'
),
Input.spend(claim)
)
def get_abandon(self, tx):
claim = Transaction(tx[0].serialize()).outputs[0]
return self._make_tx(
Output.pay_pubkey_hash(claim.amount, b'abc'),
Input.spend(claim)
)
def get_support(self, tx, amount):
claim = Transaction(tx[0].serialize()).outputs[0]
return self._make_tx(
Output.pay_support_pubkey_hash(
amount, claim.claim_name, claim.claim_id, b'abc'
)
)
def get_controlling(self):
for claim in self.sql.execute("select claim.* from claimtrie natural join claim"):
txo = self._txos[claim['txo_hash']]
controlling = txo.claim.stream.title, claim['amount'], claim['effective_amount'], claim['activation_height']
return controlling
def get_active(self):
controlling = self.get_controlling()
active = []
for claim in self.sql.execute(
f"select * from claim where activation_height <= {self._current_height}"):
txo = self._txos[claim['txo_hash']]
if controlling and controlling[0] == txo.claim.stream.title:
continue
active.append((txo.claim.stream.title, claim['amount'], claim['effective_amount'], claim['activation_height']))
return active
def get_accepted(self):
accepted = []
for claim in self.sql.execute(
f"select * from claim where activation_height > {self._current_height}"):
txo = self._txos[claim['txo_hash']]
accepted.append((txo.claim.stream.title, claim['amount'], claim['effective_amount'], claim['activation_height']))
return accepted
def advance(self, height, txs):
self._current_height = height
self.sql.advance_txs(height, txs, {'timestamp': 1}, self.daemon_height, self.timer)
return [otx[0].tx.outputs[0] for otx in txs]
def state(self, controlling=None, active=None, accepted=None):
self.assertEqual(controlling, self.get_controlling())
self.assertEqual(active or [], self.get_active())
self.assertEqual(accepted or [], self.get_accepted())
class TestClaimtrie(TestSQLDB):
def test_example_from_spec(self):
# https://spec.lbry.com/#claim-activation-example
advance, state = self.advance, self.state
stream = self.get_stream('Claim A', 10*COIN)
advance(13, [stream])
state(
controlling=('Claim A', 10*COIN, 10*COIN, 13),
active=[],
accepted=[]
)
advance(1001, [self.get_stream('Claim B', 20*COIN)])
state(
controlling=('Claim A', 10*COIN, 10*COIN, 13),
active=[],
accepted=[('Claim B', 20*COIN, 0, 1031)]
)
advance(1010, [self.get_support(stream, 14*COIN)])
state(
controlling=('Claim A', 10*COIN, 24*COIN, 13),
active=[],
accepted=[('Claim B', 20*COIN, 0, 1031)]
)
advance(1020, [self.get_stream('Claim C', 50*COIN)])
state(
controlling=('Claim A', 10*COIN, 24*COIN, 13),
active=[],
accepted=[
('Claim B', 20*COIN, 0, 1031),
('Claim C', 50*COIN, 0, 1051)]
)
advance(1031, [])
state(
controlling=('Claim A', 10*COIN, 24*COIN, 13),
active=[('Claim B', 20*COIN, 20*COIN, 1031)],
accepted=[('Claim C', 50*COIN, 0, 1051)]
)
advance(1040, [self.get_stream('Claim D', 300*COIN)])
state(
controlling=('Claim A', 10*COIN, 24*COIN, 13),
active=[('Claim B', 20*COIN, 20*COIN, 1031)],
accepted=[
('Claim C', 50*COIN, 0, 1051),
('Claim D', 300*COIN, 0, 1072)]
)
advance(1051, [])
state(
controlling=('Claim D', 300*COIN, 300*COIN, 1051),
active=[
('Claim A', 10*COIN, 24*COIN, 13),
('Claim B', 20*COIN, 20*COIN, 1031),
('Claim C', 50*COIN, 50*COIN, 1051)],
accepted=[]
)
# beyond example
advance(1052, [self.get_stream_update(stream, 290*COIN)])
state(
controlling=('Claim A', 290*COIN, 304*COIN, 13),
active=[
('Claim B', 20*COIN, 20*COIN, 1031),
('Claim C', 50*COIN, 50*COIN, 1051),
('Claim D', 300*COIN, 300*COIN, 1051),
],
accepted=[]
)
def test_competing_claims_subsequent_blocks_height_wins(self):
advance, state = self.advance, self.state
advance(13, [self.get_stream('Claim A', 10*COIN)])
state(
controlling=('Claim A', 10*COIN, 10*COIN, 13),
active=[],
accepted=[]
)
advance(14, [self.get_stream('Claim B', 10*COIN)])
state(
controlling=('Claim A', 10*COIN, 10*COIN, 13),
active=[('Claim B', 10*COIN, 10*COIN, 14)],
accepted=[]
)
advance(15, [self.get_stream('Claim C', 10*COIN)])
state(
controlling=('Claim A', 10*COIN, 10*COIN, 13),
active=[
('Claim B', 10*COIN, 10*COIN, 14),
('Claim C', 10*COIN, 10*COIN, 15)],
accepted=[]
)
def test_competing_claims_in_single_block_position_wins(self):
advance, state = self.advance, self.state
stream = self.get_stream('Claim A', 10*COIN)
stream2 = self.get_stream('Claim B', 10*COIN)
advance(13, [stream, stream2])
state(
controlling=('Claim A', 10*COIN, 10*COIN, 13),
active=[('Claim B', 10*COIN, 10*COIN, 13)],
accepted=[]
)
def test_competing_claims_in_single_block_effective_amount_wins(self):
advance, state = self.advance, self.state
stream = self.get_stream('Claim A', 10*COIN)
stream2 = self.get_stream('Claim B', 11*COIN)
advance(13, [stream, stream2])
state(
controlling=('Claim B', 11*COIN, 11*COIN, 13),
active=[('Claim A', 10*COIN, 10*COIN, 13)],
accepted=[]
)
def test_winning_claim_deleted(self):
advance, state = self.advance, self.state
stream = self.get_stream('Claim A', 10*COIN)
stream2 = self.get_stream('Claim B', 11*COIN)
advance(13, [stream, stream2])
state(
controlling=('Claim B', 11*COIN, 11*COIN, 13),
active=[('Claim A', 10*COIN, 10*COIN, 13)],
accepted=[]
)
advance(14, [self.get_abandon(stream2)])
state(
controlling=('Claim A', 10*COIN, 10*COIN, 13),
active=[],
accepted=[]
)
def test_winning_claim_deleted_and_new_claim_becomes_winner(self):
advance, state = self.advance, self.state
stream = self.get_stream('Claim A', 10*COIN)
stream2 = self.get_stream('Claim B', 11*COIN)
advance(13, [stream, stream2])
state(
controlling=('Claim B', 11*COIN, 11*COIN, 13),
active=[('Claim A', 10*COIN, 10*COIN, 13)],
accepted=[]
)
advance(15, [self.get_abandon(stream2), self.get_stream('Claim C', 12*COIN)])
state(
controlling=('Claim C', 12*COIN, 12*COIN, 15),
active=[('Claim A', 10*COIN, 10*COIN, 13)],
accepted=[]
)
def test_winning_claim_expires_and_another_takes_over(self):
advance, state = self.advance, self.state
advance(10, [self.get_stream('Claim A', 11*COIN)])
advance(20, [self.get_stream('Claim B', 10*COIN)])
state(
controlling=('Claim A', 11*COIN, 11*COIN, 10),
active=[('Claim B', 10*COIN, 10*COIN, 20)],
accepted=[]
)
advance(262984, [])
state(
controlling=('Claim B', 10*COIN, 10*COIN, 20),
active=[],
accepted=[]
)
advance(262994, [])
state(
controlling=None,
active=[],
accepted=[]
)
@staticmethod
def _get_x_with_claim_id_prefix(getter, prefix, cached_iteration=None, **kwargs):
iterations = cached_iteration+1 if cached_iteration else 100
for i in range(cached_iteration or 1, iterations):
stream = getter(f'claim #{i}', COIN, **kwargs)
if stream[0].tx.outputs[0].claim_id.startswith(prefix):
cached_iteration is None and print(f'Found "{prefix}" in {i} iterations.')
return stream
if cached_iteration:
raise ValueError(f'Failed to find "{prefix}" at cached iteration, run with None to find iteration.')
raise ValueError(f'Failed to find "{prefix}" in {iterations} iterations, try different values.')
def get_channel_with_claim_id_prefix(self, prefix, cached_iteration=None, **kwargs):
return self._get_x_with_claim_id_prefix(self.get_channel, prefix, cached_iteration, **kwargs)
def get_stream_with_claim_id_prefix(self, prefix, cached_iteration=None, **kwargs):
return self._get_x_with_claim_id_prefix(self.get_stream, prefix, cached_iteration, **kwargs)
def test_canonical_url_and_channel_validation(self):
advance = self.advance
tx_chan_a = self.get_channel_with_claim_id_prefix('a', 1, key=b'c')
tx_chan_ab = self.get_channel_with_claim_id_prefix('ab', 72, key=b'c')
txo_chan_a = tx_chan_a[0].tx.outputs[0]
advance(1, [tx_chan_a])
advance(2, [tx_chan_ab])
r_ab, r_a = self.sql._search(order_by=['creation_height'], limit=2)
self.assertEqual("@foo#a", r_a['short_url'])
self.assertEqual("@foo#ab", r_ab['short_url'])
self.assertIsNone(r_a['canonical_url'])
self.assertIsNone(r_ab['canonical_url'])
self.assertEqual(0, r_a['claims_in_channel'])
self.assertEqual(0, r_ab['claims_in_channel'])
tx_a = self.get_stream_with_claim_id_prefix('a', 2)
tx_ab = self.get_stream_with_claim_id_prefix('ab', 42)
tx_abc = self.get_stream_with_claim_id_prefix('abc', 65)
advance(3, [tx_a])
advance(4, [tx_ab, tx_abc])
r_abc, r_ab, r_a = self.sql._search(order_by=['creation_height', 'tx_position'], limit=3)
self.assertEqual("foo#a", r_a['short_url'])
self.assertEqual("foo#ab", r_ab['short_url'])
self.assertEqual("foo#abc", r_abc['short_url'])
self.assertIsNone(r_a['canonical_url'])
self.assertIsNone(r_ab['canonical_url'])
self.assertIsNone(r_abc['canonical_url'])
tx_a2 = self.get_stream_with_claim_id_prefix('a', 7, channel=txo_chan_a)
tx_ab2 = self.get_stream_with_claim_id_prefix('ab', 23, channel=txo_chan_a)
a2_claim_id = tx_a2[0].tx.outputs[0].claim_id
ab2_claim_id = tx_ab2[0].tx.outputs[0].claim_id
advance(6, [tx_a2])
advance(7, [tx_ab2])
r_ab2, r_a2 = self.sql._search(order_by=['creation_height'], limit=2)
self.assertEqual(f"foo#{a2_claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo#{ab2_claim_id[:4]}", r_ab2['short_url'])
self.assertEqual("@foo#a/foo#a", r_a2['canonical_url'])
self.assertEqual("@foo#a/foo#ab", r_ab2['canonical_url'])
self.assertEqual(2, self.sql._search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel'])
# change channel public key, invaliding stream claim signatures
advance(8, [self.get_channel_update(txo_chan_a, COIN, key=b'a')])
r_ab2, r_a2 = self.sql._search(order_by=['creation_height'], limit=2)
self.assertEqual(f"foo#{a2_claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo#{ab2_claim_id[:4]}", r_ab2['short_url'])
self.assertIsNone(r_a2['canonical_url'])
self.assertIsNone(r_ab2['canonical_url'])
self.assertEqual(0, self.sql._search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel'])
# reinstate previous channel public key (previous stream claim signatures become valid again)
channel_update = self.get_channel_update(txo_chan_a, COIN, key=b'c')
advance(9, [channel_update])
r_ab2, r_a2 = self.sql._search(order_by=['creation_height'], limit=2)
self.assertEqual(f"foo#{a2_claim_id[:2]}", r_a2['short_url'])
self.assertEqual(f"foo#{ab2_claim_id[:4]}", r_ab2['short_url'])
self.assertEqual("@foo#a/foo#a", r_a2['canonical_url'])
self.assertEqual("@foo#a/foo#ab", r_ab2['canonical_url'])
self.assertEqual(2, self.sql._search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel'])
# claim abandon updates claims_in_channel
advance(10, [self.get_abandon(tx_ab2)])
self.assertEqual(1, self.sql._search(claim_id=txo_chan_a.claim_id, limit=1)[0]['claims_in_channel'])
# delete channel, invaliding stream claim signatures
advance(11, [self.get_abandon(channel_update)])
r_a2, = self.sql._search(order_by=['creation_height'], limit=1)
self.assertEqual(f"foo#{a2_claim_id[:2]}", r_a2['short_url'])
self.assertIsNone(r_a2['canonical_url'])
def test_canonical_find_shortest_id(self):
new_hash = 'abcdef0123456789beef'
other0 = '1bcdef0123456789beef'
other1 = 'ab1def0123456789beef'
other2 = 'abc1ef0123456789beef'
other3 = 'abcdef0123456789bee1'
f = FindShortestID()
f.step(other0, new_hash)
self.assertEqual('#a', f.finalize())
f.step(other1, new_hash)
self.assertEqual('#abc', f.finalize())
f.step(other2, new_hash)
self.assertEqual('#abcd', f.finalize())
f.step(other3, new_hash)
self.assertEqual('#abcdef0123456789beef', f.finalize())
class TestTrending(TestSQLDB):
def test_trending(self):
advance, state = self.advance, self.state
no_trend = self.get_stream('Claim A', COIN)
downwards = self.get_stream('Claim B', COIN)
up_small = self.get_stream('Claim C', COIN)
up_medium = self.get_stream('Claim D', COIN)
up_biggly = self.get_stream('Claim E', COIN)
claims = advance(1, [up_biggly, up_medium, up_small, no_trend, downwards])
for window in range(1, 8):
advance(TRENDING_WINDOW * window, [
self.get_support(downwards, (20-window)*COIN),
self.get_support(up_small, int(20+(window/10)*COIN)),
self.get_support(up_medium, (20+(window*(2 if window == 7 else 1)))*COIN),
self.get_support(up_biggly, (20+(window*(3 if window == 7 else 1)))*COIN),
])
results = self.sql._search(order_by=['trending_local'])
self.assertEqual([c.claim_id for c in claims], [hexlify(c['claim_hash'][::-1]).decode() for c in results])
self.assertEqual([10, 6, 2, 0, -2], [int(c['trending_local']) for c in results])
self.assertEqual([53, 38, -32, 0, -6], [int(c['trending_global']) for c in results])
self.assertEqual([4, 4, 2, 0, 1], [int(c['trending_group']) for c in results])
self.assertEqual([53, 38, 2, 0, -6], [int(c['trending_mixed']) for c in results])
+4
View File
@@ -0,0 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.37.0rc6"
version = tuple(__version__.split('.'))
@@ -1,4 +1,4 @@
from lbry.cryptoutils import get_lbry_hash_obj
from lbrynet.cryptoutils import get_lbry_hash_obj
MAX_BLOB_SIZE = 2 * 2 ** 20
@@ -10,12 +10,12 @@ from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
from lbry.cryptoutils import backend, get_lbry_hash_obj
from lbry.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
from lbrynet.cryptoutils import backend, get_lbry_hash_obj
from lbrynet.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
from lbry.blob import MAX_BLOB_SIZE, blobhash_length
from lbry.blob.blob_info import BlobInfo
from lbry.blob.writer import HashBlobWriter
from lbrynet.blob import MAX_BLOB_SIZE, blobhash_length
from lbrynet.blob.blob_info import BlobInfo
from lbrynet.blob.writer import HashBlobWriter
log = logging.getLogger(__name__)
File renamed without changes.
@@ -2,15 +2,13 @@ import os
import typing
import asyncio
import logging
from lbry.utils import LRUCache
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbry.stream.descriptor import StreamDescriptor
from lbry.connection_manager import ConnectionManager
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbrynet.stream.descriptor import StreamDescriptor
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.dht.protocol.data_store import DictDataStore
from lbry.extras.daemon.storage import SQLiteStorage
from lbrynet.conf import Config
from lbrynet.dht.protocol.data_store import DictDataStore
from lbrynet.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
@@ -32,9 +30,6 @@ class BlobManager:
else self._node_data_store.completed_blobs
self.blobs: typing.Dict[str, AbstractBlob] = {}
self.config = config
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
self.config.blob_lru_cache_size)
self.connection_manager = ConnectionManager(loop)
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
if self.config.save_blobs:
@@ -86,11 +81,9 @@ class BlobManager:
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
if to_add:
self.completed_blob_hashes.update(to_add)
self.connection_manager.start()
return True
def stop(self):
self.connection_manager.stop()
while self.blobs:
_, blob = self.blobs.popitem()
blob.close()
@@ -2,8 +2,8 @@ import typing
import logging
import asyncio
from io import BytesIO
from lbry.error import InvalidBlobHashError, InvalidDataError
from lbry.cryptoutils import get_lbry_hash_obj
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from lbrynet.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
File renamed without changes.
@@ -2,26 +2,24 @@ import asyncio
import logging
import typing
import binascii
from lbry.error import InvalidBlobHashError, InvalidDataError
from lbry.blob_exchange.serialization import BlobResponse, BlobRequest
from lbry.utils import cache_concurrent
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
from lbrynet.utils import cache_concurrent
if typing.TYPE_CHECKING:
from lbry.blob.blob_file import AbstractBlob
from lbry.blob.writer import HashBlobWriter
from lbry.connection_manager import ConnectionManager
from lbrynet.blob.blob_file import AbstractBlob
from lbrynet.blob.writer import HashBlobWriter
log = logging.getLogger(__name__)
class BlobExchangeClientProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10,
connection_manager: typing.Optional['ConnectionManager'] = None):
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10):
self.loop = loop
self.peer_port: typing.Optional[int] = None
self.peer_address: typing.Optional[str] = None
self.transport: typing.Optional[asyncio.Transport] = None
self.peer_timeout = peer_timeout
self.connection_manager = connection_manager
self.transport: typing.Optional[asyncio.Transport] = None
self.writer: typing.Optional['HashBlobWriter'] = None
self.blob: typing.Optional['AbstractBlob'] = None
@@ -33,14 +31,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.closed = asyncio.Event(loop=self.loop)
def data_received(self, data: bytes):
if self.connection_manager:
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
# assert self.peer_address is not None
self.connection_manager.received_data(f"{self.peer_address}:{self.peer_port}", len(data))
#log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
# self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
if not self.transport or self.transport.is_closing():
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
binascii.hexlify(data))
@@ -102,15 +94,10 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
"""
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
blob_hash = self.blob.blob_hash
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
try:
msg = request.serialize()
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
self.transport.write(msg)
if self.connection_manager:
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
availability_response = response.get_availability_response()
price_response = response.get_price_response()
@@ -125,16 +112,11 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.peer_port)
log.warning(response.to_dict())
return self._blob_bytes_received, self.close()
elif availability_response and availability_response.available_blobs and \
elif availability_response.available_blobs and \
availability_response.available_blobs != [self.blob.blob_hash]:
log.warning("blob availability response doesn't match our request from %s:%i",
self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
elif not availability_response:
log.warning("response from %s:%i did not include an availability response (we requested %s)",
self.peer_address, self.peer_port, blob_hash)
return self._blob_bytes_received, self.close()
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
@@ -204,16 +186,11 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.writer = None
def connection_made(self, transport: asyncio.Transport):
addr = transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr[0], addr[1]
self.transport = transport
if self.connection_manager:
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
self.peer_address, self.peer_port = self.transport.get_extra_info('peername')
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
def connection_lost(self, reason):
if self.connection_manager:
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
str(type(reason)))
self.close()
@@ -222,19 +199,16 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
@cache_concurrent
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
peer_connect_timeout: float, blob_download_timeout: float,
connected_transport: asyncio.Transport = None, connection_id: int = 0,
connection_manager: typing.Optional['ConnectionManager'] = None)\
connected_transport: asyncio.Transport = None, connection_id: int = 0)\
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
Returns [<downloaded blob>, <keep connection>]
"""
protocol = BlobExchangeClientProtocol(
loop, blob_download_timeout, connection_manager
)
protocol = BlobExchangeClientProtocol(loop, blob_download_timeout)
if connected_transport and not connected_transport.is_closing():
connected_transport.set_protocol(protocol)
protocol.transport = connected_transport
protocol.connection_made(connected_transport)
log.debug("reusing connection for %s:%d", address, tcp_port)
else:
connected_transport = None
@@ -1,20 +1,21 @@
import asyncio
import typing
import logging
from lbry.utils import cache_concurrent
from lbry.blob_exchange.client import request_blob
from lbrynet.utils import cache_concurrent
from lbrynet.blob_exchange.client import request_blob
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.dht.node import Node
from lbry.dht.peer import KademliaPeer
from lbry.blob.blob_manager import BlobManager
from lbry.blob.blob_file import AbstractBlob
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.blob_file import AbstractBlob
log = logging.getLogger(__name__)
class BlobDownloader:
BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here
BAN_CAP = 30.0*60.0 # maximum time a node remains banned before being retried
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
peer_queue: asyncio.Queue):
@@ -37,13 +38,12 @@ class BlobDownloader:
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
if blob.get_is_verified():
return
self.scores[peer] = self.scores.get(peer, 0) - 1 # starts losing score, to account for cancelled ones
transport = self.connections.get(peer)
start = self.loop.time()
bytes_received, transport = await request_blob(
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
connection_manager=self.blob_manager.connection_manager
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id
)
if not transport and peer not in self.ignored:
self.ignored[peer] = self.loop.time()
@@ -56,24 +56,23 @@ class BlobDownloader:
self.failures[peer] = 0
self.connections[peer] = transport
elapsed = self.loop.time() - start
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 0
async def new_peer_or_finished(self):
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
def cleanup_active(self):
if not self.active_connections and not self.connections:
self.clearbanned()
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
for peer in to_remove:
del self.active_connections[peer]
self.clearbanned()
def clearbanned(self):
now = self.loop.time()
self.ignored = dict((
(peer, when) for (peer, when) in self.ignored.items()
if (now - when) < min(30.0, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
if (now - when) < min(self.BAN_CAP, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
))
@cache_concurrent
@@ -3,12 +3,12 @@ import binascii
import logging
import typing
from json.decoder import JSONDecodeError
from lbry.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
from lbry.blob_exchange.serialization import BlobAvailabilityResponse, BlobPriceResponse, BlobDownloadResponse, \
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPriceResponse, BlobDownloadResponse, \
BlobPaymentAddressResponse
if typing.TYPE_CHECKING:
from lbry.blob.blob_manager import BlobManager
from lbrynet.blob.blob_manager import BlobManager
log = logging.getLogger(__name__)
@@ -22,23 +22,15 @@ class BlobServerProtocol(asyncio.Protocol):
self.buf = b''
self.transport = None
self.lbrycrd_address = lbrycrd_address
self.peer_address_and_port: typing.Optional[str] = None
def connection_made(self, transport):
self.transport = transport
self.peer_address_and_port = "%s:%i" % self.transport.get_extra_info('peername')
self.blob_manager.connection_manager.connection_received(self.peer_address_and_port)
def connection_lost(self, exc: typing.Optional[Exception]) -> None:
self.blob_manager.connection_manager.incoming_connection_lost(self.peer_address_and_port)
def send_response(self, responses: typing.List[blob_response_types]):
to_send = []
while responses:
to_send.append(responses.pop())
serialized = BlobResponse(to_send).serialize()
self.transport.write(serialized)
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, len(serialized))
self.transport.write(BlobResponse(to_send).serialize())
async def handle_request(self, request: BlobRequest):
addr = self.transport.get_extra_info('peername')
@@ -80,7 +72,6 @@ class BlobServerProtocol(asyncio.Protocol):
def data_received(self, data):
request = None
if data:
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
message, separator, remainder = data.rpartition(b'}')
if not separator:
self.buf += data
@@ -106,7 +97,7 @@ class BlobServer:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: typing.Optional[asyncio.Task] = None
self.server_task: asyncio.Task = None
self.started_listening = asyncio.Event(loop=self.loop)
self.lbrycrd_address = lbrycrd_address
self.server_protocol_class = BlobServerProtocol
@@ -1,3 +1,2 @@
# don't touch this. CI server changes this during build/deployment
BUILD = "dev"
BUILD_COMMIT = "source installation"
Loaded 100 of 384 files, more files were not shown because too many files have changed in this diff. Show more