Compare commits

..
Author SHA1 Message Date
Victor Shyba 3a0affe8ba show status while downloading headers 2019-06-11 02:25:49 -03:00
397 changed files with 4887 additions and 41179 deletions

No files matched your search

+1 -1
View File
@@ -2,6 +2,6 @@
.tox
__pycache__
dist
lbry.egg-info
lbrynet.egg-info
docs
tests
File renamed without changes.
+74
View File
@@ -0,0 +1,74 @@
<!--
Thanks for reporting an issue to LBRY and helping us improve!
To make it possible for us to help you, please fill out below information carefully.
Before reporting any issues, please make sure that you're using the latest version.
- App: https://github.com/lbryio/lbry-desktop/releases
- Daemon: https://github.com/lbryio/lbry/releases
We are also available on Discord at https://chat.lbry.com
-->
## The Issue
In order to <achieve some value>,
as a <type of user>,
I want <some functionality>.
### Steps to reproduce
1.
2.
3.
### Expected behaviour
Tell us what should happen
### Actual behaviour
Tell us what happens instead
## System Configuration
<!-- For the app, this info is in the About section at the bottom of the Help page.
You can include a screenshot instead of typing it out -->
<!-- For the daemon, run:
curl 'http://localhost:5279' --data '{"method":"version"}'
and include the full output -->
- LBRY Daemon version:
- LBRY App version:
- LBRY Installation ID:
- Operating system:
## Anything Else
<!-- Include anything else that does not fit into the above sections -->
## Screenshots
<!-- If a screenshot would help explain the bug, please include one or two here -->
## Internal Use
### Acceptance Criteria
1.
2.
3.
### Definition of Done
- [ ] Tested against acceptance criteria
- [ ] Tested against the assumptions of user story
- [ ] The project builds without errors
- [ ] Unit tests are written and passing
- [ ] Tests on devices/browsers listed in the issue have passed
- [ ] QA performed & issues resolved
- [ ] Refactoring completed
- [ ] Any configuration or build changes documented
- [ ] Documentation updated
- [ ] Peer Code Review performed
+38
View File
@@ -0,0 +1,38 @@
## PR Checklist
Please check all that apply to this PR using "x":
- [ ] I have checked that this PR is not a duplicate of an existing PR (open, closed or merged)
- [ ] I have checked that this PR does not introduce a breaking change
- [ ] This PR introduces breaking changes and I have provided a detailed explanation below
## PR Type
What kind of change does this PR introduce?
Why is this change necessary?
<!-- Please check all that apply to this PR using "x". -->
- [ ] Bugfix
- [ ] Feature
- [ ] Breaking changes (bugfix or feature that introduces breaking changes)
- [ ] Code style update (formatting)
- [ ] Refactoring (no functional changes)
- [ ] Documentation changes
- [ ] Other - Please describe:
## Fixes
Issue Number: N/A
## What is the current behavior?
## What is the new behavior?
## Other information
<!-- If this PR contains a breaking change, please describe the impact and solution strategy for existing applications below. -->
+12 -1
View File
@@ -1,2 +1,13 @@
.DS_Store
/build
/dist
/.tox
/.idea
/.DS_Store
/.coverage
/lbry-venv
lbrynet.egg-info
__pycache__
_trial_temp/
/tests/integration/files
+5 -1
View File
@@ -24,7 +24,7 @@ persistent=yes
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=4
jobs=1
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
@@ -77,6 +77,8 @@ disable=
dangerous-default-value,
duplicate-code,
fixme,
global-statement,
inherit-non-class,
invalid-name,
len-as-condition,
locally-disabled,
@@ -109,11 +111,13 @@ disable=
unnecessary-lambda,
unused-argument,
unused-variable,
wildcard-import,
wrong-import-order,
wrong-import-position,
deprecated-lambda,
simplifiable-if-statement,
unidiomatic-typecheck,
global-at-module-level,
inconsistent-return-statements,
keyword-arg-before-vararg,
assignment-from-no-return,
+83 -84
View File
@@ -3,144 +3,143 @@ dist: xenial
language: python
python: "3.7"
env:
global:
# must also be updated in wine_build.sh
- TORBA=master
jobs:
include:
- stage: code quality
name: "pylint & mypy"
name: "pylint lbrynet"
install:
- make install
script: make lint
- pip install astroid pylint
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script: pylint lbrynet
- stage: test
name: "LBRY Unit Tests"
name: "Unit Tests"
install:
- make install
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script:
- cd lbry && HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
after_success:
- coverage combine lbry/
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- name: "LBRY Integration Tests"
- name: "Integration Tests"
install:
- pip install coverage tox-travis
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
script: cd lbry && tox
after_success:
- coverage combine lbry
- bash <(curl -s https://codecov.io/bash)
- pip install tox-travis coverage
- pushd .. && git clone --single-branch --branch ${TORBA} https://github.com/lbryio/torba.git && popd
- &torba-tests
name: "Torba Unit Tests"
env: TESTTYPE=unit
install:
- pip install coverage tox-travis
script: cd torba && tox
script: tox
after_success:
- coverage combine torba/tests
- coverage combine tests/
- bash <(curl -s https://codecov.io/bash)
- <<: *torba-tests
name: "Torba Integration Tests"
env: TESTTYPE=integration
- name: "Run Examples"
install:
- make install
script:
- cd lbry && HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
after_success:
- coverage combine lbry
- bash <(curl -s https://codecov.io/bash)
- &build
stage: build
name: "Linux"
env: OS=linux
install:
- pip install pyinstaller awscli
- cd torba && pip install -e . && cd ..
- cd lbry
- python scripts/set_build.py
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script:
- pyinstaller -F -n lbrynet lbry/extras/cli.py
- cd dist
- chmod +x lbrynet
- zip -j lbrynet-${OS}.zip lbrynet
- shasum -a 256 -b lbrynet-${OS}.zip
- ./lbrynet --version
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
after_success:
- aws configure set aws_access_key_id $ARTIFACTS_KEY
- aws configure set aws_secret_access_key $ARTIFACTS_SECRET
- aws configure set region us-east-1
- export S3_PATH="daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})"
- aws s3 cp lbrynet-${OS}.zip s3://build.lbry.io/${S3_PATH}/lbrynet-${OS}.zip
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- stage: build
name: "Windows"
language: generic
services:
- docker
install:
- docker pull lbry/pyinstaller34_32bits:py371
script:
- python scripts/set_build.py
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
deploy:
provider: releases
api_key:
secure: "unnR+aSJ1937Cl1PyBBZzGuZvV5W5TGcXELhXTgyOeeI6FgO/j80qmbNxJDA7qdFH/hvVicQFWoflhZu2dxN5rYP5BQJW3q3XoOLY3XAc1s1vicFkwqn3TIfdFiJTz+/D9eBUBBhHKeYFxm3M+thvklTLgjKl6fflh14NfGuNTevK9yQke8wewW3f9UmFTo1qNOPF1OsTZRbwua6oQYa59P+KukoPt4Dsu1VtILtTkj7hfEsUL79cjotwO3gkhYftxbl/xeDSZWOt+9Nhb8ZKmQG/uDx4JiTMm5lWRk4QB7pUujZ1CftxCYWz/lJx9nuJpdCOgP624tcHymErNlD+vGLwMTNslcXGYkAJH6xvGyxBJ+Obc8vRVnZbRM26BfH34TcPK1ueRxHSrDUbzMIIUsgcoZAxBuim8uDPp+K7bGqiygzSs2vQfr9U5Jhe9/F8sPdtNctfJZEfgmthNTeVFjyNsGIfIt754uGSfACqM7wDLh6fbKx7M+FHlNyOdvYCrbKUOAYXmTikYIpVDvlaaeMO+N+uW8Rhvm1j+JU7CVwhMavLySaPVc6Dt5OxiMMmxw9mVrjW9bBPjS5AkrS5MOA13T5wapoLzH6+gE92U4HzA6ilMcwRaQPSFnK2JU7tzyt2Wy1PH4MjHowXI2WyICG1x510dD3tX1P/1px8ro="
file: lbrynet-${OS}.zip
api_key: $GITHUB_OAUTH_TOKEN
file: dist/lbrynet-windows.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
paths:
- lbrynet-windows.zip
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- &build
name: "Linux"
env: OS=linux
install:
- pip3 install pyinstaller
- pip3 install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- python3 scripts/set_build.py
- pip3 install -e .
script:
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
- chmod +x dist/lbrynet
- zip -j dist/lbrynet-${OS}.zip dist/lbrynet
- ./dist/lbrynet --version
deploy:
provider: releases
api_key: $GITHUB_OAUTH_TOKEN
file: dist/lbrynet-${OS}.zip
skip_cleanup: true
overwrite: true
draft: true
on:
tags: true
addons:
artifacts:
working_dir: dist
paths:
- lbrynet-${OS}.zip
# artifact uploader thinks lbrynet is a directory, https://github.com/travis-ci/artifacts/issues/78
target_paths:
- /daemon/build-${TRAVIS_BUILD_NUMBER}_commit-${TRAVIS_COMMIT:0:7}_branch-${TRAVIS_BRANCH}$([ ! -z ${TRAVIS_TAG} ] && echo _tag-${TRAVIS_TAG})
- <<: *build
name: "Mac"
os: osx
osx_image: xcode8.3
language: shell
language: generic
env: OS=mac
cache: false
before_install:
- brew upgrade python || true
- pip3 install --user --upgrade pip virtualenv
- /Users/travis/Library/Python/3.7/bin/virtualenv $HOME/venv
- source $HOME/venv/bin/activate
before_cache:
- brew cleanup
- <<: *build
name: "Windows"
os: windows
language: shell
env:
- OS=windows
- PATH=/c/Python37:/c/Python37/Scripts:/C/Windows/System32/downlevel:$PATH
before_install:
- choco install python --version=3.7.4 --x86
- python -m pip install --upgrade pip
- pip install pywin32
script:
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
- cd dist
- 7z a -tzip lbrynet-windows.zip lbrynet.exe
- sha256sum -b lbrynet-windows.zip
- ./lbrynet.exe --version
- brew upgrade python || true
- if: tag IS present
stage: build
name: "Wallet Server Docker Image - Tagged Release"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f lbry/scripts/Dockerfile.wallet_server .
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:$TRAVIS_TAG
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
stage: build
name: "Wallet Server Docker Image - Master"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:master -f lbry/scripts/Dockerfile.wallet_server .
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:master
cache:
directories:
- $HOME/venv
- $HOME/.cache/pip
- $HOME/Library/Caches/pip
- $HOME/Library/Caches/Homebrew
- $TRAVIS_BUILD_DIR/.tox
+6
View File
@@ -0,0 +1,6 @@
# Add a CHANGELOG entry for app changes
has_app_changes = !git.modified_files.grep(/lbrynet/).empty?
if !git.modified_files.include?("CHANGELOG.md") && has_app_changes
fail("Please include a CHANGELOG entry.")
message "See http://keepachangelog.com/en/0.3.0/ for details on good changelog guidelines"
end
+19 -50
View File
@@ -1,6 +1,6 @@
# Installing LBRY
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md)!
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md).
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
@@ -9,7 +9,7 @@ Here's a video walkthrough of this setup, which is itself hosted by the LBRY net
## Prerequisites
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/)
After installing python 3, you'll need to install some additional libraries depending on your operating system.
@@ -17,7 +17,7 @@ After installing python 3, you'll need to install some additional libraries depe
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
These environment variables also need to be set:
These environment variables also need to be set
1. PYTHONUNBUFFERED=1
2. EVENT_NOKQUEUE=1
@@ -27,16 +27,14 @@ Remaining dependencies can then be installed by running:
brew install python protobuf
```
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/.
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
### Linux
On Ubuntu (16.04 minimum, we recommend 18.04), install the following:
On Ubuntu (we recommend 18.04), install the following:
```
sudo add-apt-repository ppa:deadsnakes/ppa
sudo apt-get update
sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
```
On Raspbian, you will also need to install `python-pyparsing`.
@@ -45,61 +43,32 @@ If you're running another Linux distro, install the equivalent of the above pack
## Installation
### Linux/Mac
To install on Linux/Mac:
To install:
```
Clone the repository:
$ git clone https://github.com/lbryio/lbry-sdk.git
$ cd lbry-sdk
git clone https://github.com/lbryio/lbry.git
cd lbry
Create a Python virtual environment for lbry-sdk:
$ python3.7 -m venv lbry-venv
Creating venv:
python -m venv lbry-venv
Activating lbry-sdk virtual environment:
$ source lbry-venv/bin/activate
Activating lbry-venv on Linux/Mac:
source lbry-venv/bin/activate
Make sure you're on Python 3.7+ (as the default Python in virtual environment):
$ python --version
Activating lbry-venv on Windows:
lbry-venv\Scripts\activate
Install packages:
$ make install
python --version # Python 2 is not supported. Make sure you're on Python 3.7
If you are on Linux and using PyCharm, generates initial configs:
$ make idea
pip install -e .
```
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `python3.7 -m venv lbry-venv` command.
### Windows
To install on Windows:
```
Clone the repository:
> git clone https://github.com/lbryio/lbry-sdk.git
> cd lbry-sdk
Create a Python virtual environment for lbry-sdk:
> python -m venv lbry-venv
Activating lbry-sdk virtual environment:
> lbry-venv\Scripts\activate
Install packages:
> cd torba
> pip install -e .
> cd ../lbry
> pip install -e .
```
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `virtualenv` command.
## Run the tests
To run the unit tests from the repo directory:
```
python -m unittest discover -s lbry tests.unit
trial --reactor=asyncio tests.unit
```
## Usage
-18
View File
@@ -1,18 +0,0 @@
install:
cd torba && pip install -e .
cd lbry && pip install -e .
pip install mypy==0.701
pip install coverage astroid pylint
lint:
cd lbry && pylint lbry
cd torba && pylint --rcfile=setup.cfg torba
cd torba && mypy --ignore-missing-imports torba
test:
cd lbry && tox
cd torba && tox
idea:
mkdir -p .idea
cp -r lbry/scripts/idea/* .idea
+9 -9
View File
@@ -1,19 +1,19 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry-sdk.svg?branch=master)](https://travis-ci.org/lbryio/lbry-sdk) [![Test Coverage](https://codecov.io/gh/lbryio/lbry-sdk/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry-sdk)
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
* Built on Python 3.7+ and `asyncio`.
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/schema)).
* Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/extras/daemon)).
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
* Wallet implementation for the LBRY blockchain ([lbrynet.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
## Installation
Our [releases page](https://github.com/lbryio/lbry-sdk/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
Our [releases page](https://github.com/lbryio/lbry/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
## Usage
@@ -53,4 +53,4 @@ The documentation for the API can be found [here](https://lbry.tech/api/sdk).
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/lbry/example_daemon_settings.yml).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry/blob/master/example_daemon_settings.yml).
File renamed without changes.
File diff suppressed because it is too large. Load diff
File renamed without changes.

Before

Width:  |  Height:  |  Size: 521 B

After

Width:  |  Height:  |  Size: 521 B

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.4 KiB

After

Width:  |  Height:  |  Size: 1.4 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.6 KiB

After

Width:  |  Height:  |  Size: 1.6 KiB

File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.

Before

Width:  |  Height:  |  Size: 7.4 KiB

After

Width:  |  Height:  |  Size: 7.4 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 6.1 KiB

After

Width:  |  Height:  |  Size: 6.1 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 97 KiB

After

Width:  |  Height:  |  Size: 97 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 361 KiB

After

Width:  |  Height:  |  Size: 361 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 5.3 KiB

After

Width:  |  Height:  |  Size: 5.3 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 31 KiB

After

Width:  |  Height:  |  Size: 31 KiB

View File
File renamed without changes.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

-13
View File
@@ -1,13 +0,0 @@
.DS_Store
/build
/dist
/.tox
/.idea
/.coverage
/lbry-venv
lbry.egg-info
__pycache__
_trial_temp/
/tests/integration/files
-56
View File
@@ -1,56 +0,0 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry-sdk.svg?branch=master)](https://travis-ci.org/lbryio/lbry-sdk) [![Test Coverage](https://codecov.io/gh/lbryio/lbry-sdk/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry-sdk)
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
* Built on Python 3.7+ and `asyncio`.
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/schema)).
* Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/lbry/extras/daemon)).
## Installation
Our [releases page](https://github.com/lbryio/lbry-sdk/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
## Usage
Run `lbrynet start` to launch the API server.
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
With the daemon running, `lbrynet commands` will show you a list of commands.
The full API is documented [here](https://lbry.tech/api/sdk).
## Running from source
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
## Contributing
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
## License
This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
## Contact
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
## Additional information and links
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/lbry/example_daemon_settings.yml).
-4
View File
@@ -1,4 +0,0 @@
__name__ = "lbry"
__version__ = "0.43.2"
version = tuple(__version__.split('.'))
-1
View File
@@ -1 +0,0 @@
from lbry.extras.daemon import Components # register Component classes
-66
View File
@@ -1,66 +0,0 @@
import logging
import time
import hashlib
import binascii
from lbry import utils
import ecdsa
from torba.client.hash import sha256
from lbry.wallet.transaction import Output
log = logging.getLogger(__name__)
def get_encoded_signature(signature):
signature = signature.encode() if type(signature) is str else signature
r = int(signature[:int(len(signature) / 2)], 16)
s = int(signature[int(len(signature) / 2):], 16)
return ecdsa.util.sigencode_der(r, s, len(signature) * 4)
def cid2hash(claim_id: str) -> bytes:
return binascii.unhexlify(claim_id.encode())[::-1]
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
if type(channel) is Output:
try:
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [
comment['signing_ts'].encode(),
cid2hash(comment['channel_id']),
signing_field.encode()
]
return Output.is_signature_valid(
get_encoded_signature(comment['signature']),
sha256(b''.join(pieces)),
channel.claim.channel.public_key_bytes
)
except KeyError:
pass
return False
def sign_comment(comment: dict, channel: Output, abandon=False):
timestamp = str(int(time.time()))
signing_field = comment['comment_id'] if abandon else comment['comment']
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
digest = sha256(b''.join(pieces))
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
comment.update({
'signature': binascii.hexlify(signature).decode(),
'signing_ts': timestamp
})
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
params = params or {}
params.update(kwargs)
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
async with utils.aiohttp_request('POST', url, json=json_body) as response:
try:
result = await response.json()
return result['result'] if 'result' in result else result
except Exception as cte:
log.exception('Unable to decode response from server: %s', cte)
return await response.text()
-13
View File
@@ -1,13 +0,0 @@
from typing import List
import re
MULTI_SPACE_RE = re.compile(r"\s{2,}")
WEIRD_CHARS_RE = re.compile(r"[#!~]")
def normalize_tag(tag: str):
return MULTI_SPACE_RE.sub(' ', WEIRD_CHARS_RE.sub(' ', tag.lower().replace("'", ""))).strip()
def clean_tags(tags: List[str]):
return [tag for tag in set(normalize_tag(tag) for tag in tags) if tag]
-11
View File
@@ -1,11 +0,0 @@
__node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.2.1/lbrycrd-linux.zip'
)
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
from lbry.wallet.manager import LbryWalletManager
from lbry.wallet.network import Network
from lbry.wallet.ledger import MainNetLedger, RegTestLedger, TestNetLedger
-5
View File
@@ -1,5 +0,0 @@
TXO_TYPES = {
"stream": 1,
"channel": 2,
"support": 3
}
-284
View File
@@ -1,284 +0,0 @@
import asyncio
import logging
from binascii import unhexlify
from functools import partial
from typing import Tuple, List
from datetime import datetime
from torba.client.baseledger import BaseLedger
from torba.client.baseaccount import SingleKey
from lbry.schema.result import Outputs
from lbry.schema.url import URL
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet.account import Account
from lbry.wallet.network import Network
from lbry.wallet.database import WalletDatabase
from lbry.wallet.transaction import Transaction, Output
from lbry.wallet.header import Headers, UnvalidatedHeaders
log = logging.getLogger(__name__)
class MainNetLedger(BaseLedger):
name = 'LBRY Credits'
symbol = 'LBC'
network_name = 'mainnet'
headers: Headers
account_class = Account
database_class = WalletDatabase
headers_class = Headers
network_class = Network
transaction_class = Transaction
db: WalletDatabase
secret_prefix = bytes((0x1c,))
pubkey_address_prefix = bytes((0x55,))
script_address_prefix = bytes((0x7a,))
extended_public_key_prefix = unhexlify('0488b21e')
extended_private_key_prefix = unhexlify('0488ade4')
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
genesis_bits = 0x1f00ffff
target_timespan = 150
default_fee_per_byte = 50
default_fee_per_name_char = 200000
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
async def _inflate_outputs(self, query):
outputs = Outputs.from_base64(await query)
txs = []
if len(outputs.txs) > 0:
txs = await asyncio.gather(*(self.cache_transaction(*tx) for tx in outputs.txs))
return outputs.inflate(txs), outputs.offset, outputs.total
async def resolve(self, urls):
resolve = partial(self.network.retriable_call, self.network.resolve)
txos = (await self._inflate_outputs(resolve(urls)))[0]
assert len(urls) == len(txos), "Mismatch between urls requested for resolve and responses received."
result = {}
for url, txo in zip(urls, txos):
if txo and URL.parse(url).has_stream_in_channel:
if not txo.channel or not txo.is_signed_by(txo.channel, self):
txo = None
if txo:
result[url] = txo
else:
result[url] = {'error': f'{url} did not resolve to a claim'}
return result
async def claim_search(self, **kwargs) -> Tuple[List[Output], int, int]:
return await self._inflate_outputs(self.network.claim_search(**kwargs))
async def get_claim_by_claim_id(self, claim_id) -> Output:
for claim in (await self.claim_search(claim_id=claim_id))[0]:
return claim
async def start(self):
await super().start()
await asyncio.gather(*(a.maybe_migrate_certificates() for a in self.accounts))
await asyncio.gather(*(a.save_max_gap() for a in self.accounts))
await self._report_state()
async def _report_state(self):
try:
for account in self.accounts:
balance = dewies_to_lbc(await account.get_balance())
channel_count = await account.get_channel_count()
claim_count = await account.get_claim_count()
if isinstance(account.receiving, SingleKey):
log.info("Loaded single key account %s with %s LBC. "
"%d channels, %d certificates and %d claims",
account.id, balance, channel_count, len(account.channel_keys), claim_count)
else:
total_receiving = len((await account.receiving.get_addresses()))
total_change = len((await account.change.get_addresses()))
log.info("Loaded account %s with %s LBC, %d receiving addresses (gap: %d), "
"%d change addresses (gap: %d), %d channels, %d certificates and %d claims. ",
account.id, balance, total_receiving, account.receiving.gap, total_change,
account.change.gap, channel_count, len(account.channel_keys), claim_count)
except:
log.exception(
'Failed to display wallet state, please file issue '
'for this bug along with the traceback you see below:')
@staticmethod
def constraint_spending_utxos(constraints):
constraints['txo_type'] = 0
def get_utxos(self, **constraints):
self.constraint_spending_utxos(constraints)
return super().get_utxos(**constraints)
def get_utxo_count(self, **constraints):
self.constraint_spending_utxos(constraints)
return super().get_utxo_count(**constraints)
def get_claims(self, **constraints):
return self.db.get_claims(**constraints)
def get_claim_count(self, **constraints):
return self.db.get_claim_count(**constraints)
def get_streams(self, **constraints):
return self.db.get_streams(**constraints)
def get_stream_count(self, **constraints):
return self.db.get_stream_count(**constraints)
def get_channels(self, **constraints):
return self.db.get_channels(**constraints)
def get_channel_count(self, **constraints):
return self.db.get_channel_count(**constraints)
def get_supports(self, **constraints):
return self.db.get_supports(**constraints)
def get_support_count(self, **constraints):
return self.db.get_support_count(**constraints)
async def get_transaction_history(self, **constraints):
txs = await self.db.get_transactions(**constraints)
headers = self.headers
history = []
for tx in txs:
ts = headers[tx.height]['timestamp'] if tx.height > 0 else None
item = {
'txid': tx.id,
'timestamp': ts,
'date': datetime.fromtimestamp(ts).isoformat(' ')[:-3] if tx.height > 0 else None,
'confirmations': (headers.height+1) - tx.height if tx.height > 0 else 0,
'claim_info': [],
'update_info': [],
'support_info': [],
'abandon_info': []
}
is_my_inputs = all([txi.is_my_account for txi in tx.inputs])
if is_my_inputs:
# fees only matter if we are the ones paying them
item['value'] = dewies_to_lbc(tx.net_account_balance+tx.fee)
item['fee'] = dewies_to_lbc(-tx.fee)
else:
# someone else paid the fees
item['value'] = dewies_to_lbc(tx.net_account_balance)
item['fee'] = '0.0'
for txo in tx.my_claim_outputs:
item['claim_info'].append({
'address': txo.get_address(self),
'balance_delta': dewies_to_lbc(-txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position
})
for txo in tx.my_update_outputs:
if is_my_inputs: # updating my own claim
previous = None
for txi in tx.inputs:
if txi.txo_ref.txo is not None:
other_txo = txi.txo_ref.txo
if (other_txo.is_claim or other_txo.script.is_support_claim) \
and other_txo.claim_id == txo.claim_id:
previous = other_txo
break
if previous is not None:
item['update_info'].append({
'address': txo.get_address(self),
'balance_delta': dewies_to_lbc(previous.amount-txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position
})
else: # someone sent us their claim
item['update_info'].append({
'address': txo.get_address(self),
'balance_delta': dewies_to_lbc(0),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position
})
for txo in tx.my_support_outputs:
item['support_info'].append({
'address': txo.get_address(self),
'balance_delta': dewies_to_lbc(txo.amount if not is_my_inputs else -txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'is_tip': not is_my_inputs,
'nout': txo.position
})
if is_my_inputs:
for txo in tx.other_support_outputs:
item['support_info'].append({
'address': txo.get_address(self),
'balance_delta': dewies_to_lbc(-txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'is_tip': is_my_inputs,
'nout': txo.position
})
for txo in tx.my_abandon_outputs:
item['abandon_info'].append({
'address': txo.get_address(self),
'balance_delta': dewies_to_lbc(txo.amount),
'amount': dewies_to_lbc(txo.amount),
'claim_id': txo.claim_id,
'claim_name': txo.claim_name,
'nout': txo.position
})
history.append(item)
return history
def get_transaction_history_count(self, **constraints):
return self.db.get_transaction_count(**constraints)
@staticmethod
async def get_detailed_balance(accounts, confirmations=0, reserved_subtotals=False):
result = {}
for account in accounts:
balance = await account.get_detailed_balance(confirmations, reserved_subtotals)
if result:
for key, value in balance.items():
if key == 'reserved_subtotals':
if value is not None:
for subkey, subvalue in value.items():
result['reserved_subtotals'][subkey] += subvalue
else:
result[key] += value
else:
result = balance
return result
class TestNetLedger(MainNetLedger):
network_name = 'testnet'
pubkey_address_prefix = bytes((111,))
script_address_prefix = bytes((196,))
extended_public_key_prefix = unhexlify('043587cf')
extended_private_key_prefix = unhexlify('04358394')
class RegTestLedger(MainNetLedger):
network_name = 'regtest'
headers_class = UnvalidatedHeaders
pubkey_address_prefix = bytes((111,))
script_address_prefix = bytes((196,))
extended_public_key_prefix = unhexlify('043587cf')
extended_private_key_prefix = unhexlify('04358394')
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
genesis_bits = 0x207fffff
target_timespan = 1
-220
View File
@@ -1,220 +0,0 @@
CLAIM_TYPES = {
'stream': 1,
'channel': 2,
}
STREAM_TYPES = {
'video': 1,
'audio': 2,
'image': 3,
'document': 4,
'binary': 5,
'model': 6
}
MATURE_TAGS = [
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
]
COMMON_TAGS = {
"gaming": "gaming",
"people & blogs": "people_and_blogs",
"pop culture": "pop_culture",
"entertainment": "entertainment",
"technology": "technology",
"music": "music",
"funny": "funny",
"education": "education",
"learning": "learning",
"news": "news",
"gameplay": "gameplay",
"science & technology": "science_and_technology",
"playstation 4": "playstation_4",
"beliefs": "beliefs",
"nature": "nature",
"news & politics": "news_and_politics",
"comedy": "comedy",
"games": "games",
"sony interactive entertainment": "sony_interactive_entertainment",
"film & animation": "film_and_animation",
"game": "game",
"howto & style": "howto_and_style",
"weapons": "weapons",
"blockchain": "blockchain",
"video game": "video_game",
"sports": "sports",
"walkthrough": "walkthrough",
"ps4live": "ps4live",
"art": "art",
"pc": "pc",
"economics": "economics",
"automotive": "automotive",
"minecraft": "minecraft",
"playthrough": "playthrough",
"ps4share": "ps4share",
"tutorial": "tutorial",
"play": "play",
"twitch": "twitch",
"how to": "how_to",
"ps4": "ps4",
"bitcoin": "bitcoin",
"fortnite": "fortnite",
"commentary": "commentary",
"lets play": "lets_play",
"fun": "fun",
"politics": "politics",
"xbox": "xbox",
"autos & vehicles": "autos_and_vehicles",
"travel & events": "travel_and_events",
"food": "food",
"science": "science",
"mature": "mature",
"xbox one": "xbox_one",
"liberal": "liberal",
"democrat": "democrat",
"progressive": "progressive",
"survival": "survival",
"nonprofits & activism": "nonprofits_and_activism",
"cryptocurrency": "cryptocurrency",
"playstation": "playstation",
"nintendo": "nintendo",
"government": "government",
"steam": "steam",
"podcast": "podcast",
"horror": "horror",
"conservative": "conservative",
"reaction": "reaction",
"trailer": "trailer",
"love": "love",
"cnn": "cnn",
"republican": "republican",
"gamer": "gamer",
"political": "political",
"hangoutsonair": "hangoutsonair",
"hoa": "hoa",
"msnbc": "msnbc",
"cbs": "cbs",
"donald trump": "donald_trump",
"fiction": "fiction",
"fox news": "fox_news",
"anime": "anime",
"crypto": "crypto",
"ethereum": "ethereum",
"call of duty": "call_of_duty",
"multiplayer": "multiplayer",
"android": "android",
"epic": "epic",
"rpg": "rpg",
"adventure": "adventure",
"secular talk": "secular_talk",
"btc": "btc",
"atheist": "atheist",
"atheism": "atheism",
"ps3": "ps3",
"video games": "video_games",
"cod": "cod",
"agnostic": "agnostic",
"movie": "movie",
"online": "online",
"fps": "fps",
"mod": "mod",
"reviews": "reviews",
"sharefactory": "sharefactory",
"world": "world",
"space": "space",
"hilarious": "hilarious",
"stream": "stream",
"lol": "lol",
"sony": "sony",
"god": "god",
"lets": "lets",
"dance": "dance",
"pvp": "pvp",
"tech": "tech",
"zombies": "zombies",
"pokemon": "pokemon",
"fail": "fail",
"xbox 360": "xbox_360",
"film": "film",
"unboxing": "unboxing",
"animation": "animation",
"travel": "travel",
"money": "money",
"wwe": "wwe",
"how": "how",
"mods": "mods",
"pubg": "pubg",
"indie": "indie",
"strategy": "strategy",
"history": "history",
"rap": "rap",
"ios": "ios",
"sony computer entertainment": "sony_computer_entertainment",
"mobile": "mobile",
"trump": "trump",
"flat earth": "flat_earth",
"hack": "hack",
"trap": "trap",
"fox": "fox",
"vlogging": "vlogging",
"news radio": "news_radio",
"humor": "humor",
"facebook": "facebook",
"edm": "edm",
"fitness": "fitness",
"vaping": "vaping",
"hip hop": "hip_hop",
"secular": "secular",
"jesus": "jesus",
"vape": "vape",
"song": "song",
"remix": "remix",
"guitar": "guitar",
"daily": "daily",
"mining": "mining",
"diy": "diy",
"videogame": "videogame",
"pets & animals": "pets_and_animals",
"funny moments": "funny_moments",
"religion": "religion",
"death": "death",
"media": "media",
"nbc": "nbc",
"war": "war",
"freedom": "freedom",
"viral": "viral",
"meme": "meme",
"family": "family",
"gold": "gold",
"photography": "photography",
"chill": "chill",
"zombie": "zombie",
"computer": "computer",
"sniper": "sniper",
"bible": "bible",
"linux": "linux",
"overwatch": "overwatch",
"pro": "pro",
"dragon": "dragon",
"litecoin": "litecoin",
"gta": "gta",
"iphone": "iphone",
"house": "house",
"bass": "bass",
"bitcoin news": "bitcoin_news",
"wii": "wii",
"crash": "crash",
"league of legends": "league_of_legends",
"grand theft auto v": "grand_theft_auto_v",
"mario": "mario",
"mmorpg": "mmorpg",
"satire": "satire",
"fire": "fire",
"racing": "racing",
"apple": "apple",
"health": "health",
"instrumental": "instrumental",
"destiny": "destiny",
"truth": "truth",
"race": "race"
}
-531
View File
@@ -1,531 +0,0 @@
import time
import struct
import sqlite3
import logging
from operator import itemgetter
from typing import Tuple, List, Dict, Union, Type, Optional
from binascii import unhexlify
from decimal import Decimal
from contextvars import ContextVar
from functools import wraps
from dataclasses import dataclass
from torba.client.basedatabase import query, interpolate
from lbry.schema.url import URL, normalize_name
from lbry.schema.tags import clean_tags
from lbry.schema.result import Outputs
from lbry.wallet.ledger import BaseLedger, MainNetLedger, RegTestLedger
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
class SQLiteOperationalError(sqlite3.OperationalError):
def __init__(self, metrics):
super().__init__('sqlite query errored')
self.metrics = metrics
class SQLiteInterruptedError(sqlite3.OperationalError):
def __init__(self, metrics):
super().__init__('sqlite query interrupted')
self.metrics = metrics
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
INTEGER_PARAMS = {
'height', 'creation_height', 'activation_height', 'expiration_height',
'timestamp', 'creation_timestamp', 'release_time', 'fee_amount',
'tx_position', 'channel_join',
'amount', 'effective_amount', 'support_amount',
'trending_group', 'trending_mixed',
'trending_local', 'trending_global',
}
SEARCH_PARAMS = {
'name', 'claim_id', 'txid', 'nout', 'channel', 'channel_ids', 'not_channel_ids',
'public_key_id', 'claim_type', 'stream_types', 'media_types', 'fee_currency',
'has_channel_signature', 'signature_valid',
'any_tags', 'all_tags', 'not_tags',
'any_locations', 'all_locations', 'not_locations',
'any_languages', 'all_languages', 'not_languages',
'is_controlling', 'limit', 'offset', 'order_by',
'no_totals',
} | INTEGER_PARAMS
ORDER_FIELDS = {
'name', 'claim_hash'
} | INTEGER_PARAMS
PRAGMAS = """
pragma journal_mode=WAL;
"""
@dataclass
class ReaderState:
db: sqlite3.Connection
stack: List[List]
metrics: Dict
is_tracking_metrics: bool
ledger: Type[BaseLedger]
query_timeout: float
log: logging.Logger
def close(self):
self.db.close()
def reset_metrics(self):
self.stack = []
self.metrics = {}
def set_query_timeout(self):
stop_at = time.perf_counter() + self.query_timeout
def interruptor():
if time.perf_counter() >= stop_at:
self.db.interrupt()
return
self.db.set_progress_handler(interruptor, 100)
ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
def initializer(log, _path, _ledger_name, query_timeout, _measure=False):
db = sqlite3.connect(_path, isolation_level=None, uri=True)
db.row_factory = sqlite3.Row
ctx.set(
ReaderState(
db=db, stack=[], metrics={}, is_tracking_metrics=_measure,
ledger=MainNetLedger if _ledger_name == 'mainnet' else RegTestLedger,
query_timeout=query_timeout, log=log
)
)
def cleanup():
ctx.get().close()
ctx.set(None)
def measure(func):
@wraps(func)
def wrapper(*args, **kwargs):
state = ctx.get()
if not state.is_tracking_metrics:
return func(*args, **kwargs)
metric = {}
state.metrics.setdefault(func.__name__, []).append(metric)
state.stack.append([])
start = time.perf_counter()
try:
return func(*args, **kwargs)
finally:
elapsed = int((time.perf_counter()-start)*1000)
metric['total'] = elapsed
metric['isolated'] = (elapsed-sum(state.stack.pop()))
if state.stack:
state.stack[-1].append(elapsed)
return wrapper
def reports_metrics(func):
@wraps(func)
def wrapper(*args, **kwargs):
state = ctx.get()
if not state.is_tracking_metrics:
return func(*args, **kwargs)
state.reset_metrics()
r = func(*args, **kwargs)
return r, state.metrics
return wrapper
@reports_metrics
def search_to_bytes(constraints) -> Union[bytes, Tuple[bytes, Dict]]:
return encode_result(search(constraints))
@reports_metrics
def resolve_to_bytes(urls) -> Union[bytes, Tuple[bytes, Dict]]:
return encode_result(resolve(urls))
def encode_result(result):
return Outputs.to_bytes(*result)
@measure
def execute_query(sql, values) -> List:
context = ctx.get()
context.set_query_timeout()
try:
return context.db.execute(sql, values).fetchall()
except sqlite3.OperationalError as err:
plain_sql = interpolate(sql, values)
if context.is_tracking_metrics:
context.metrics['execute_query'][-1]['sql'] = plain_sql
if str(err) == "interrupted":
context.log.warning("interrupted slow sqlite query:\n%s", plain_sql)
raise SQLiteInterruptedError(context.metrics)
context.log.exception('failed running query', exc_info=err)
raise SQLiteOperationalError(context.metrics)
def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
if 'order_by' in constraints:
sql_order_by = []
for order_by in constraints['order_by']:
is_asc = order_by.startswith('^')
column = order_by[1:] if is_asc else order_by
if column not in ORDER_FIELDS:
raise NameError(f'{column} is not a valid order_by field')
if column == 'name':
column = 'normalized'
sql_order_by.append(
f"claim.{column} ASC" if is_asc else f"claim.{column} DESC"
)
constraints['order_by'] = sql_order_by
ops = {'<=': '__lte', '>=': '__gte', '<': '__lt', '>': '__gt'}
for constraint in INTEGER_PARAMS:
if constraint in constraints:
value = constraints.pop(constraint)
postfix = ''
if isinstance(value, str):
if len(value) >= 2 and value[:2] in ops:
postfix, value = ops[value[:2]], value[2:]
elif len(value) >= 1 and value[0] in ops:
postfix, value = ops[value[0]], value[1:]
if constraint == 'fee_amount':
value = Decimal(value)*1000
constraints[f'claim.{constraint}{postfix}'] = int(value)
if constraints.pop('is_controlling', False):
if {'sequence', 'amount_order'}.isdisjoint(constraints):
for_count = False
constraints['claimtrie.claim_hash__is_not_null'] = ''
if 'sequence' in constraints:
constraints['order_by'] = 'claim.activation_height ASC'
constraints['offset'] = int(constraints.pop('sequence')) - 1
constraints['limit'] = 1
if 'amount_order' in constraints:
constraints['order_by'] = 'claim.effective_amount DESC'
constraints['offset'] = int(constraints.pop('amount_order')) - 1
constraints['limit'] = 1
if 'claim_id' in constraints:
claim_id = constraints.pop('claim_id')
if len(claim_id) == 40:
constraints['claim.claim_id'] = claim_id
else:
constraints['claim.claim_id__like'] = f'{claim_id[:40]}%'
if 'name' in constraints:
constraints['claim.normalized'] = normalize_name(constraints.pop('name'))
if 'public_key_id' in constraints:
constraints['claim.public_key_hash'] = sqlite3.Binary(
ctx.get().ledger.address_to_hash160(constraints.pop('public_key_id')))
if 'channel_hash' in constraints:
constraints['claim.channel_hash'] = sqlite3.Binary(constraints.pop('channel_hash'))
if 'channel_ids' in constraints:
channel_ids = constraints.pop('channel_ids')
if channel_ids:
constraints['claim.channel_hash__in'] = [
sqlite3.Binary(unhexlify(cid)[::-1]) for cid in channel_ids
]
if 'not_channel_ids' in constraints:
not_channel_ids = constraints.pop('not_channel_ids')
if not_channel_ids:
not_channel_ids_binary = [
sqlite3.Binary(unhexlify(ncid)[::-1]) for ncid in not_channel_ids
]
if constraints.get('has_channel_signature', False):
constraints['claim.channel_hash__not_in'] = not_channel_ids_binary
else:
constraints['null_or_not_channel__or'] = {
'claim.signature_valid__is_null': True,
'claim.channel_hash__not_in': not_channel_ids_binary
}
if 'signature_valid' in constraints:
has_channel_signature = constraints.pop('has_channel_signature', False)
if has_channel_signature:
constraints['claim.signature_valid'] = constraints.pop('signature_valid')
else:
constraints['null_or_signature__or'] = {
'claim.signature_valid__is_null': True,
'claim.signature_valid': constraints.pop('signature_valid')
}
elif constraints.pop('has_channel_signature', False):
constraints['claim.signature_valid__is_not_null'] = True
if 'txid' in constraints:
tx_hash = unhexlify(constraints.pop('txid'))[::-1]
nout = constraints.pop('nout', 0)
constraints['claim.txo_hash'] = sqlite3.Binary(
tx_hash + struct.pack('<I', nout)
)
if 'claim_type' in constraints:
constraints['claim.claim_type'] = CLAIM_TYPES[constraints.pop('claim_type')]
if 'stream_types' in constraints:
stream_types = constraints.pop('stream_types')
if stream_types:
constraints['claim.stream_type__in'] = [
STREAM_TYPES[stream_type] for stream_type in stream_types
]
if 'media_types' in constraints:
media_types = constraints.pop('media_types')
if media_types:
constraints['claim.media_type__in'] = media_types
if 'fee_currency' in constraints:
constraints['claim.fee_currency'] = constraints.pop('fee_currency').lower()
_apply_constraints_for_array_attributes(constraints, 'tag', clean_tags, for_count)
_apply_constraints_for_array_attributes(constraints, 'language', lambda _: _, for_count)
_apply_constraints_for_array_attributes(constraints, 'location', lambda _: _, for_count)
select = f"SELECT {cols} FROM claim"
sql, values = query(
select if for_count else select+"""
LEFT JOIN claimtrie USING (claim_hash)
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
""", **constraints
)
return sql, values
def get_claims(cols, for_count=False, **constraints) -> List:
if 'channel' in constraints:
channel_url = constraints.pop('channel')
match = resolve_url(channel_url)
if isinstance(match, sqlite3.Row):
constraints['channel_hash'] = match['claim_hash']
else:
return [[0]] if cols == 'count(*)' else []
sql, values = _get_claims(cols, for_count, **constraints)
return execute_query(sql, values)
@measure
def get_claims_count(**constraints) -> int:
constraints.pop('offset', None)
constraints.pop('limit', None)
constraints.pop('order_by', None)
count = get_claims('count(*)', for_count=True, **constraints)
return count[0][0]
@measure
def search(constraints) -> Tuple[List, List, int, int]:
assert set(constraints).issubset(SEARCH_PARAMS), \
f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
total = None
if not constraints.pop('no_totals', False):
total = get_claims_count(**constraints)
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
if 'order_by' not in constraints:
constraints['order_by'] = ["claim_hash"]
txo_rows = _search(**constraints)
channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows)))
extra_txo_rows = []
if channel_hashes:
extra_txo_rows = _search(
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}
)
return txo_rows, extra_txo_rows, constraints['offset'], total
def _search(**constraints):
return get_claims(
"""
claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
claim.claim_hash, claim.txo_hash,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed,
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.signature_valid
""", **constraints
)
@measure
def resolve(urls) -> Tuple[List, List]:
result = []
channel_hashes = set()
for raw_url in urls:
match = resolve_url(raw_url)
result.append(match)
if isinstance(match, sqlite3.Row) and match['channel_hash']:
channel_hashes.add(match['channel_hash'])
extra_txo_rows = []
if channel_hashes:
extra_txo_rows = _search(
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}
)
return result, extra_txo_rows
@measure
def resolve_url(raw_url):
try:
url = URL.parse(raw_url)
except ValueError as e:
return e
channel = None
if url.has_channel:
query = url.channel.to_dict()
if set(query) == {'name'}:
query['is_controlling'] = True
else:
query['order_by'] = ['^height']
matches = _search(**query, limit=1)
if matches:
channel = matches[0]
else:
return LookupError(f'Could not find channel in "{raw_url}".')
if url.has_stream:
query = url.stream.to_dict()
if channel is not None:
if set(query) == {'name'}:
# temporarily emulate is_controlling for claims in channel
query['order_by'] = ['effective_amount', '^height']
else:
query['order_by'] = ['^channel_join']
query['channel_hash'] = channel['claim_hash']
query['signature_valid'] = 1
elif set(query) == {'name'}:
query['is_controlling'] = 1
matches = _search(**query, limit=1)
if matches:
return matches[0]
else:
return LookupError(f'Could not find stream in "{raw_url}".')
return channel
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
not_items = set(cleaner(constraints.pop(f'not_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
all_items = {item for item in all_items if item not in not_items}
any_items = {item for item in any_items if item not in not_items}
any_queries = {}
if attr == 'tag':
common_tags = any_items & COMMON_TAGS.keys()
if common_tags:
any_items -= common_tags
if len(common_tags) < 5:
for item in common_tags:
index_name = COMMON_TAGS[item]
any_queries[f'#_common_tag_{index_name}'] = f"""
EXISTS(
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
AND tag = '{item}'
)
"""
elif len(common_tags) >= 5:
constraints.update({
f'$any_common_tag{i}': item for i, item in enumerate(common_tags)
})
values = ', '.join(
f':$any_common_tag{i}' for i in range(len(common_tags))
)
any_queries[f'#_any_common_tags'] = f"""
EXISTS(
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
AND tag IN ({values})
)
"""
if any_items:
constraints.update({
f'$any_{attr}{i}': item for i, item in enumerate(any_items)
})
values = ', '.join(
f':$any_{attr}{i}' for i in range(len(any_items))
)
if for_count or attr == 'tag':
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
"""
else:
any_queries[f'#_any_{attr}'] = f"""
EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
"""
if len(any_queries) == 1:
constraints.update(any_queries)
elif len(any_queries) > 1:
constraints[f'ORed_{attr}_queries__any'] = any_queries
if all_items:
constraints[f'$all_{attr}_count'] = len(all_items)
constraints.update({
f'$all_{attr}{i}': item for i, item in enumerate(all_items)
})
values = ', '.join(
f':$all_{attr}{i}' for i in range(len(all_items))
)
if for_count:
constraints[f'claim.claim_hash__in#_all_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
GROUP BY claim_hash HAVING COUNT({attr}) = :$all_{attr}_count
"""
else:
constraints[f'#_all_{attr}'] = f"""
{len(all_items)}=(
SELECT count(*) FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
"""
if not_items:
constraints.update({
f'$not_{attr}{i}': item for i, item in enumerate(not_items)
})
values = ', '.join(
f':$not_{attr}{i}' for i in range(len(not_items))
)
if for_count:
constraints[f'claim.claim_hash__not_in#_not_{attr}'] = f"""
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
"""
else:
constraints[f'#_not_{attr}'] = f"""
NOT EXISTS(
SELECT 1 FROM {attr} WHERE
claim.claim_hash={attr}.claim_hash
AND {attr} IN ({values})
)
"""
-135
View File
@@ -1,135 +0,0 @@
import time
import math
from typing import Tuple
def calculate_elapsed(start) -> int:
return int((time.perf_counter() - start) * 1000)
def calculate_avg_percentiles(data) -> Tuple[int, int, int, int, int, int, int, int]:
if not data:
return 0, 0, 0, 0, 0, 0, 0, 0
data.sort()
size = len(data)
return (
int(sum(data) / size),
data[0],
data[math.ceil(size * .05) - 1],
data[math.ceil(size * .25) - 1],
data[math.ceil(size * .50) - 1],
data[math.ceil(size * .75) - 1],
data[math.ceil(size * .95) - 1],
data[-1]
)
def remove_select_list(sql) -> str:
return sql[sql.index('FROM'):]
class APICallMetrics:
def __init__(self, name):
self.name = name
# total requests received
self.receive_count = 0
self.cache_response_count = 0
# millisecond timings for query based responses
self.query_response_times = []
self.query_intrp_times = []
self.query_error_times = []
self.query_python_times = []
self.query_wait_times = []
self.query_sql_times = [] # aggregate total of multiple SQL calls made per request
self.individual_sql_times = [] # every SQL query run on server
# actual queries
self.errored_queries = set()
self.interrupted_queries = set()
def to_json(self):
return {
# total requests received
"receive_count": self.receive_count,
# sum of these is total responses made
"cache_response_count": self.cache_response_count,
"query_response_count": len(self.query_response_times),
"intrp_response_count": len(self.query_intrp_times),
"error_response_count": len(self.query_error_times),
# millisecond timings for non-cache responses
"response": calculate_avg_percentiles(self.query_response_times),
"interrupt": calculate_avg_percentiles(self.query_intrp_times),
"error": calculate_avg_percentiles(self.query_error_times),
# response, interrupt and error each also report the python, wait and sql stats:
"python": calculate_avg_percentiles(self.query_python_times),
"wait": calculate_avg_percentiles(self.query_wait_times),
"sql": calculate_avg_percentiles(self.query_sql_times),
# extended timings for individual sql executions
"individual_sql": calculate_avg_percentiles(self.individual_sql_times),
"individual_sql_count": len(self.individual_sql_times),
# actual queries
"errored_queries": list(self.errored_queries),
"interrupted_queries": list(self.interrupted_queries),
}
def start(self):
self.receive_count += 1
def cache_response(self):
self.cache_response_count += 1
def _add_query_timings(self, request_total_time, metrics):
if metrics and 'execute_query' in metrics:
sub_process_total = metrics[self.name][0]['total']
individual_query_times = [f['total'] for f in metrics['execute_query']]
aggregated_query_time = sum(individual_query_times)
self.individual_sql_times.extend(individual_query_times)
self.query_sql_times.append(aggregated_query_time)
self.query_python_times.append(sub_process_total - aggregated_query_time)
self.query_wait_times.append(request_total_time - sub_process_total)
@staticmethod
def _add_queries(query_set, metrics):
if metrics and 'execute_query' in metrics:
for execute_query in metrics['execute_query']:
if 'sql' in execute_query:
query_set.add(remove_select_list(execute_query['sql']))
def query_response(self, start, metrics):
self.query_response_times.append(calculate_elapsed(start))
self._add_query_timings(self.query_response_times[-1], metrics)
def query_interrupt(self, start, metrics):
self.query_intrp_times.append(calculate_elapsed(start))
self._add_queries(self.interrupted_queries, metrics)
self._add_query_timings(self.query_intrp_times[-1], metrics)
def query_error(self, start, metrics):
self.query_error_times.append(calculate_elapsed(start))
self._add_queries(self.errored_queries, metrics)
self._add_query_timings(self.query_error_times[-1], metrics)
class ServerLoadData:
def __init__(self):
self._apis = {}
def for_api(self, name) -> APICallMetrics:
if name not in self._apis:
self._apis[name] = APICallMetrics(name)
return self._apis[name]
def to_json_and_reset(self, status):
try:
return {
'api': {name: api.to_json() for name, api in self._apis.items()},
'status': status
}
finally:
self._apis = {}
-279
View File
@@ -1,279 +0,0 @@
import os
import math
import time
import base64
import asyncio
from binascii import hexlify
from pylru import lrucache
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from torba.rpc.jsonrpc import RPCError, JSONRPC
from torba.server.session import ElectrumX, SessionManager
from torba.server import util
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYDB
from lbry.wallet.server.db import reader
from lbry.wallet.server.websocket import AdminWebSocket
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
class ResultCacheItem:
__slots__ = '_result', 'lock', 'has_result'
def __init__(self):
self.has_result = asyncio.Event()
self.lock = asyncio.Lock()
self._result = None
@property
def result(self) -> str:
return self._result
@result.setter
def result(self, result: str):
self._result = result
if result is not None:
self.has_result.set()
class LBRYSessionManager(SessionManager):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.query_executor = None
self.websocket = None
self.metrics = ServerLoadData()
self.metrics_loop = None
self.running = False
if self.env.websocket_host is not None and self.env.websocket_port is not None:
self.websocket = AdminWebSocket(self)
self.search_cache = self.bp.search_cache
self.search_cache['search'] = lrucache(10000)
self.search_cache['resolve'] = lrucache(10000)
async def process_metrics(self):
while self.running:
data = self.metrics.to_json_and_reset({
'sessions': self.session_count(),
'height': self.db.db_height,
})
if self.websocket is not None:
self.websocket.send_message(data)
await asyncio.sleep(1)
async def start_other(self):
self.running = True
path = os.path.join(self.env.db_dir, 'claims.db')
args = dict(
initializer=reader.initializer,
initargs=(self.logger, path, self.env.coin.NET, self.env.database_query_timeout,
self.env.track_metrics)
)
if self.env.max_query_workers is not None and self.env.max_query_workers == 0:
self.query_executor = ThreadPoolExecutor(max_workers=1, **args)
else:
self.query_executor = ProcessPoolExecutor(
max_workers=self.env.max_query_workers or max(os.cpu_count(), 4), **args
)
if self.websocket is not None:
await self.websocket.start()
if self.env.track_metrics:
self.metrics_loop = asyncio.create_task(self.process_metrics())
async def stop_other(self):
self.running = False
if self.env.track_metrics:
self.metrics_loop.cancel()
if self.websocket is not None:
await self.websocket.stop()
self.query_executor.shutdown()
class LBRYElectrumX(ElectrumX):
PROTOCOL_MIN = (0, 0) # temporary, for supporting 0.10 protocol
max_errors = math.inf # don't disconnect people for errors! let them happen...
session_mgr: LBRYSessionManager
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# fixme: this is a rebase hack, we need to go through ChainState instead later
self.daemon = self.session_mgr.daemon
self.bp: LBRYBlockProcessor = self.session_mgr.bp
self.db: LBRYDB = self.bp.db
def set_request_handlers(self, ptuple):
super().set_request_handlers(ptuple)
handlers = {
'blockchain.transaction.get_height': self.transaction_get_height,
'blockchain.claimtrie.search': self.claimtrie_search,
'blockchain.claimtrie.resolve': self.claimtrie_resolve,
'blockchain.claimtrie.getclaimsbyids': self.claimtrie_getclaimsbyids,
'blockchain.block.get_server_height': self.get_server_height,
}
self.request_handlers.update(handlers)
def get_metrics_or_placeholder_for_api(self, query_name):
""" Do not hold on to a reference to the metrics
returned by this method past an `await` or
you may be working with a stale metrics object.
"""
if self.env.track_metrics:
return self.session_mgr.metrics.for_api(query_name)
else:
return APICallMetrics(query_name)
async def run_in_executor(self, query_name, func, kwargs):
start = time.perf_counter()
try:
result = await asyncio.get_running_loop().run_in_executor(
self.session_mgr.query_executor, func, kwargs
)
except reader.SQLiteInterruptedError as error:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.query_interrupt(start, error.metrics)
raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out')
except reader.SQLiteOperationalError as error:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.query_error(start, error.metrics)
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
except:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.query_error(start, {})
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
if self.env.track_metrics:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
(result, metrics_data) = result
metrics.query_response(start, metrics_data)
return base64.b64encode(result).decode()
async def run_and_cache_query(self, query_name, function, kwargs):
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.start()
cache = self.session_mgr.search_cache[query_name]
cache_key = str(kwargs)
cache_item = cache.get(cache_key)
if cache_item is None:
cache_item = cache[cache_key] = ResultCacheItem()
elif cache_item.result is not None:
metrics.cache_response()
return cache_item.result
async with cache_item.lock:
if cache_item.result is None:
cache_item.result = await self.run_in_executor(
query_name, function, kwargs
)
else:
metrics = self.get_metrics_or_placeholder_for_api(query_name)
metrics.cache_response()
return cache_item.result
async def claimtrie_search(self, **kwargs):
if kwargs:
return await self.run_and_cache_query('search', reader.search_to_bytes, kwargs)
async def claimtrie_resolve(self, *urls):
if urls:
return await self.run_and_cache_query('resolve', reader.resolve_to_bytes, urls)
async def get_server_height(self):
return self.bp.height
async def transaction_get_height(self, tx_hash):
self.assert_tx_hash(tx_hash)
transaction_info = await self.daemon.getrawtransaction(tx_hash, True)
if transaction_info and 'hex' in transaction_info and 'confirmations' in transaction_info:
# an unconfirmed transaction from lbrycrdd will not have a 'confirmations' field
return (self.db.db_height - transaction_info['confirmations']) + 1
elif transaction_info and 'hex' in transaction_info:
return -1
return None
async def claimtrie_getclaimsbyids(self, *claim_ids):
claims = await self.batched_formatted_claims_from_daemon(claim_ids)
return dict(zip(claim_ids, claims))
async def batched_formatted_claims_from_daemon(self, claim_ids):
claims = await self.daemon.getclaimsbyids(claim_ids)
result = []
for claim in claims:
if claim and claim.get('value'):
result.append(self.format_claim_from_daemon(claim))
return result
def format_claim_from_daemon(self, claim, name=None):
"""Changes the returned claim data to the format expected by lbry and adds missing fields."""
if not claim:
return {}
# this ISO-8859 nonsense stems from a nasty form of encoding extended characters in lbrycrd
# it will be fixed after the lbrycrd upstream merge to v17 is done
# it originated as a fear of terminals not supporting unicode. alas, they all do
if 'name' in claim:
name = claim['name'].encode('ISO-8859-1').decode()
info = self.db.sql.get_claims(claim_id=claim['claimId'])
if not info:
# raise RPCError("Lbrycrd has {} but not lbryumx, please submit a bug report.".format(claim_id))
return {}
address = info.address.decode()
# fixme: temporary
#supports = self.format_supports_from_daemon(claim.get('supports', []))
supports = []
amount = get_from_possible_keys(claim, 'amount', 'nAmount')
height = get_from_possible_keys(claim, 'height', 'nHeight')
effective_amount = get_from_possible_keys(claim, 'effective amount', 'nEffectiveAmount')
valid_at_height = get_from_possible_keys(claim, 'valid at height', 'nValidAtHeight')
result = {
"name": name,
"claim_id": claim['claimId'],
"txid": claim['txid'],
"nout": claim['n'],
"amount": amount,
"depth": self.db.db_height - height + 1,
"height": height,
"value": hexlify(claim['value'].encode('ISO-8859-1')).decode(),
"address": address, # from index
"supports": supports,
"effective_amount": effective_amount,
"valid_at_height": valid_at_height
}
if 'claim_sequence' in claim:
# TODO: ensure that lbrycrd #209 fills in this value
result['claim_sequence'] = claim['claim_sequence']
else:
result['claim_sequence'] = -1
if 'normalized_name' in claim:
result['normalized_name'] = claim['normalized_name'].encode('ISO-8859-1').decode()
return result
def assert_tx_hash(self, value):
'''Raise an RPCError if the value is not a valid transaction
hash.'''
try:
if len(util.hex_to_bytes(value)) == 32:
return
except Exception:
pass
raise RPCError(1, f'{value} should be a transaction hash')
def assert_claim_id(self, value):
'''Raise an RPCError if the value is not a valid claim id
hash.'''
try:
if len(util.hex_to_bytes(value)) == 20:
return
except Exception:
pass
raise RPCError(1, f'{value} should be a claim id hash')
def get_from_possible_keys(dictionary, *keys):
for key in keys:
if key in dictionary:
return dictionary[key]
-55
View File
@@ -1,55 +0,0 @@
import asyncio
from weakref import WeakSet
from aiohttp.web import Application, AppRunner, WebSocketResponse, TCPSite
from aiohttp.http_websocket import WSMsgType, WSCloseCode
class AdminWebSocket:
def __init__(self, manager):
self.manager = manager
self.app = Application()
self.app['websockets'] = WeakSet()
self.app.router.add_get('/', self.on_connect)
self.app.on_shutdown.append(self.on_shutdown)
self.runner = AppRunner(self.app)
async def on_status(self, _):
if not self.app['websockets']:
return
self.send_message({
'type': 'status',
'height': self.manager.daemon.cached_height(),
})
def send_message(self, msg):
for web_socket in self.app['websockets']:
asyncio.create_task(web_socket.send_json(msg))
async def start(self):
await self.runner.setup()
await TCPSite(self.runner, self.manager.env.websocket_host, self.manager.env.websocket_port).start()
async def stop(self):
await self.runner.cleanup()
async def on_connect(self, request):
web_socket = WebSocketResponse()
await web_socket.prepare(request)
self.app['websockets'].add(web_socket)
try:
async for msg in web_socket:
if msg.type == WSMsgType.TEXT:
await self.on_status(None)
elif msg.type == WSMsgType.ERROR:
print('web socket connection closed with exception %s' %
web_socket.exception())
finally:
self.app['websockets'].discard(web_socket)
return web_socket
@staticmethod
async def on_shutdown(app):
for web_socket in set(app['websockets']):
await web_socket.close(code=WSCloseCode.GOING_AWAY, message='Server shutdown')
-32
View File
@@ -1,32 +0,0 @@
import argparse
import sqlite3
from binascii import hexlify
from lbry.wallet.transaction import Output
def check(db_path, claim_id):
db = sqlite3.connect(db_path)
db.row_factory = sqlite3.Row
claim = db.execute('select * from claim where claim_id=?', (claim_id,)).fetchone()
if not claim:
print('Could not find claim.')
return
channel = db.execute('select * from claim where claim_hash=?', (claim['channel_hash'],)).fetchone()
if not channel:
print('Could not find channel for this claim.')
print(f"Claim: {claim['claim_name']}")
print(f"Channel: {channel['claim_name']}")
print(f"Signature: {hexlify(claim['signature']).decode()}")
print(f"Digest: {hexlify(claim['signature_digest']).decode()}")
print(f"Pubkey: {hexlify(channel['public_key_bytes']).decode()}")
print("Valid: {}".format(Output.is_signature_valid(
claim['signature'], claim['signature_digest'], channel['public_key_bytes']
)))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('db_path')
parser.add_argument('claim_id')
args = parser.parse_args()
check(args.db_path, args.claim_id)
-177
View File
@@ -1,177 +0,0 @@
import os
import time
import textwrap
import argparse
import asyncio
import logging
from concurrent.futures.process import ProcessPoolExecutor
from lbry.wallet.server.db.reader import search_to_bytes, initializer, _get_claims, interpolate
from lbry.wallet.ledger import MainNetLedger
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
log.setLevel(logging.CRITICAL)
DEFAULT_ANY_TAGS = [
'blockchain',
'news',
'learning',
'technology',
'automotive',
'economics',
'food',
'science',
'art',
'nature'
]
COMMON_AND_RARE = [
'gaming',
'ufos'
]
COMMON_AND_RARE2 = [
'city fix',
'gaming'
]
RARE_ANY_TAGS = [
'city fix',
'ufos',
]
CITY_FIX = [
'city fix'
]
MATURE_TAGS = [
'porn',
'nsfw',
'mature',
'xxx'
]
ORDER_BY = [
[
"trending_global",
"trending_mixed",
],
[
"release_time"
],
[
"effective_amount"
]
]
def get_args(limit=20):
args = []
any_tags_combinations = [DEFAULT_ANY_TAGS, COMMON_AND_RARE, RARE_ANY_TAGS, COMMON_AND_RARE2, CITY_FIX, []]
not_tags_combinations = [MATURE_TAGS, []]
for no_fee in [False, True]:
for claim_type in [None, 'stream', 'channel']:
for no_totals in [True]:
for offset in [0, 100]:
for any_tags in any_tags_combinations:
for not_tags in not_tags_combinations:
for order_by in ORDER_BY:
kw = {
'order_by': order_by,
'offset': offset,
'limit': limit,
'no_totals': no_totals
}
if not_tags:
kw['not_tags'] = not_tags
if any_tags:
kw['any_tags'] = any_tags
if claim_type:
kw['claim_type'] = claim_type
if no_fee:
kw['fee_amount'] = 0
args.append(kw)
print(f"-- Trying {len(args)} argument combinations")
return args
def _search(kwargs):
start = time.perf_counter()
error = None
try:
search_to_bytes(kwargs)
except Exception as err:
error = str(err)
return time.perf_counter() - start, kwargs, error
async def search(executor, kwargs):
return await asyncio.get_running_loop().run_in_executor(
executor, _search, kwargs
)
async def main(db_path, max_query_time):
args = dict(initializer=initializer, initargs=(log, db_path, MainNetLedger, 0.25))
workers = max(os.cpu_count(), 4)
log.info(f"using {workers} reader processes")
query_executor = ProcessPoolExecutor(workers, **args)
tasks = [search(query_executor, constraints) for constraints in get_args()]
try:
results = await asyncio.gather(*tasks)
query_times = [
{
'sql': interpolate(*_get_claims("""
claimtrie.claim_hash as is_controlling,
claimtrie.last_take_over_height,
claim.claim_hash, claim.txo_hash,
claim.claims_in_channel,
claim.height, claim.creation_height,
claim.activation_height, claim.expiration_height,
claim.effective_amount, claim.support_amount,
claim.trending_group, claim.trending_mixed,
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.signature_valid
""", **constraints)),
'duration': ts,
'error': error
}
for ts, constraints, error in results
]
errored = [query_info for query_info in query_times if query_info['error']]
errors = {str(query_info['error']): [] for query_info in errored}
for error in errored:
errors[str(error['error'])].append(error['sql'])
slow = [
query_info for query_info in query_times
if not query_info['error'] and query_info['duration'] > (max_query_time / 2.0)
]
fast = [
query_info for query_info in query_times
if not query_info['error'] and query_info['duration'] <= (max_query_time / 2.0)
]
print(f"-- {len(fast)} queries were fast")
slow.sort(key=lambda query_info: query_info['duration'], reverse=True)
print(f"-- Failing queries:")
for error in errors:
print(f"-- Failure: \"{error}\"")
for failing_query in errors[error]:
print(f"{textwrap.dedent(failing_query)};\n")
print()
print(f"-- Slow queries:")
for slow_query in slow:
print(f"-- Query took {slow_query['duration']}\n{textwrap.dedent(slow_query['sql'])};\n")
finally:
query_executor.shutdown()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--db_path', dest='db_path', default=os.path.expanduser('~/claims.db'), type=str)
parser.add_argument('--max_time', dest='max_time', default=0.25, type=float)
args = parser.parse_args()
db_path = args.db_path
max_query_time = args.max_time
asyncio.run(main(db_path, max_query_time))
-107
View File
@@ -1,107 +0,0 @@
import time
import asyncio
import random
from argparse import ArgumentParser
from torba.client.basenetwork import ClientSession
class AgentSmith(ClientSession):
async def do_nefarious_things(self):
await self.send_request('blockchain.claimtrie.search', {
'no_totals': True,
'offset': random.choice(range(0, 300, 20)),
'limit': 20,
'any_tags': (
random.choice([[
random.choice(['gaming', 'games', 'game']) +
random.choice(['entertainment', 'playthrough', 'funny']) +
random.choice(['xbox', 'xbox one', 'xbox news'])
], [
random.choice(['aliens', 'alien', 'ufo', 'ufos']) +
random.choice(['news', 'sighting', 'sightings'])
], [
random.choice(['art', 'automotive']),
random.choice(['blockchain', 'economics', 'food']),
random.choice(['funny', 'learnings', 'nature']),
random.choice(['news', 'science', 'technology'])
]
])
),
'not_tags': random.choice([[], [
'porn', 'mature', 'xxx', 'nsfw'
]]),
'order_by': random.choice([
['release_time'],
['trending_global', 'trending_mixed'],
['effective_amount']
])
})
class AgentSmithProgram:
def __init__(self, host, port):
self.host, self.port = host, port
self.agent_smiths = []
async def make_one_more_of_them(self):
smith = AgentSmith(network=None, server=(self.host, self.port))
await smith.create_connection()
self.agent_smiths.append(smith)
async def coordinate_nefarious_activity(self):
start = time.perf_counter()
await asyncio.gather(
*(s.do_nefarious_things() for s in self.agent_smiths),
return_exceptions=True
)
return time.perf_counter() - start
def __len__(self):
return len(self.agent_smiths)
async def delete_one_smith(self):
if self.agent_smiths:
await self.agent_smiths.pop().close()
async def delete_program(self):
await asyncio.gather(*(
s.close() for s in self.agent_smiths
))
async def main(host, port):
smiths = AgentSmithProgram(host, port)
await smiths.make_one_more_of_them()
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
ease_off = 0
for i in range(1000):
await asyncio.sleep(1)
if activity.done() and activity.result() < .9:
print('more, more, more...')
await asyncio.gather(*(
asyncio.create_task(smiths.make_one_more_of_them()) for _ in range(20)
))
else:
print('!!!!!!!!!!!!!!')
print('IS NEO LOSING?')
print('!!!!!!!!!!!!!!')
await asyncio.gather(*(
asyncio.create_task(smiths.delete_one_smith()) for _ in range(21)
))
print(f'coordinate all {len(smiths)} smiths to action')
activity = asyncio.create_task(smiths.coordinate_nefarious_activity())
print('finishing up any remaining actions')
await activity
print('neo has won, deleting agents...')
await smiths.delete_program()
print('done.')
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument('--host', dest='host', default='localhost', type=str)
parser.add_argument('--port', dest='port', default=50001, type=int)
args = parser.parse_args()
asyncio.run(main(args.host, args.port))
-17
View File
@@ -1,17 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/torba" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/torba/tests" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lbry" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lbry/tests" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/lbry/scripts" isTestSource="false" />
</content>
<orderEntry type="jdk" jdkName="Python 3.7 (lbry)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
-8
View File
@@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/lbry-sdk.iml" filepath="$PROJECT_DIR$/.idea/lbry-sdk.iml" />
</modules>
</component>
</project>
-6
View File
@@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
-49
View File
@@ -1,49 +0,0 @@
import os, asyncio, aiohttp, json, slack, sqlparse
async def listen(slack_client, url):
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(3)) as session:
print(f"connecting to {url}")
try:
ws = await session.ws_connect(url)
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
print(f"failed to connect to {url}")
return
print(f"connected to {url}")
async for msg in ws:
r = json.loads(msg.data)
try:
queries = r["api"]["search"]["interrupted_queries"]
except KeyError:
continue
for q in queries:
# clean = re.sub(r"\s+", " ", q)
clean = sqlparse.format(q, reindent=True, keyword_case='upper')
print(f'{url}: {clean}')
response = await slack_client.chat_postMessage(
username=url,
icon_emoji=":hourglass_flowing_sand:",
channel='#clubhouse-de-obscure',
text="*Query timed out:* " + clean
)
if not response["ok"]:
print("SLACK ERROR:\n", response)
print()
async def main():
try:
slack_client = slack.WebClient(token=os.environ['SLACK_TOKEN'], run_async=True)
except KeyError:
print("Error: SLACK_TOKEN env var required")
return
num_servers = 5
tasks = []
for i in range(1, num_servers+1):
tasks.append(asyncio.create_task(listen(slack_client, f'http://spv{i}.lbry.com:50005')))
await asyncio.gather(*tasks)
asyncio.run(main())
-62
View File
@@ -1,62 +0,0 @@
import uvloop, asyncio, time, sys, logging
from concurrent.futures import ProcessPoolExecutor
from lbry.wallet.server.db import reader
from lbry.wallet.server.metrics import calculate_avg_percentiles
db_path = '../../../lbryconf/wallet-server/claims.db'
default_query_timout = 0.25
log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())
async def run_times(executor, iterations, show=True):
start = time.perf_counter()
timings = await asyncio.gather(*(asyncio.get_running_loop().run_in_executor(
executor, reader.search_to_bytes, {
'no_totals': True,
'offset': 0,
'limit': 20,
'any_tags': [
'ufos', 'city fix'
],
'not_tags': [
'porn', 'mature', 'xxx', 'nsfw'
],
'order_by': [
'release_time'
]
}
) for _ in range(iterations)))
timings = [r[1]['execute_query'][0]['total'] for r in timings]
total = int((time.perf_counter() - start) * 100)
if show:
avg = sum(timings)/len(timings)
print(f"{iterations:4}: {total}ms total concurrent, {len(timings)*avg*1000:.3f}s total sequential (avg*runs)")
print(f" {total/len(timings):.1f}ms/query concurrent (total/runs)")
print(f" {avg:.1f}ms/query actual average (sum(queries)/runs)")
stats = calculate_avg_percentiles(timings)
print(f" min: {stats[1]}, 5%: {stats[2]}, 25%: {stats[3]}, 50%: {stats[4]}, 75%: {stats[5]}, 95%: {stats[6]}, max: {stats[7]}")
sys.stdout.write(' sample:')
for i, t in zip(range(10), timings[::-1]):
sys.stdout.write(f' {t}ms')
print(' ...\n' if len(timings) > 10 else '\n')
async def main():
executor = ProcessPoolExecutor(
4, initializer=reader.initializer, initargs=(log, db_path, 'mainnet', 1.0, True)
)
#await run_times(executor, 4, show=False)
#await run_times(executor, 1)
await run_times(executor, 2**3)
await run_times(executor, 2**5)
await run_times(executor, 2**7)
#await run_times(executor, 2**9)
#await run_times(executor, 2**11)
#await run_times(executor, 2**13)
executor.shutdown(True)
if __name__ == '__main__':
uvloop.install()
asyncio.run(main())
-47
View File
@@ -1,47 +0,0 @@
import asyncio
from torba.client.basenetwork import ClientSession
from torba.rpc.jsonrpc import RPCError
import logging
import json
import sys
logging.getLogger('torba').setLevel(logging.CRITICAL)
async def main():
try:
hostname = sys.argv[1]
except IndexError:
hostname = 'spv1.lbry.com'
loop = asyncio.get_event_loop()
client = ClientSession(network=None, server=(hostname, 50001))
error = None
args = {
'any_tags': ['art'],
'not_tags': ['xxx', 'porn', 'mature', 'nsfw', 'titan'],
'order_by': ["name"],
'offset': 3000,
'limit': 200,
'no_totals': False,
}
start = loop.time()
try:
await client.create_connection()
try:
await client.send_request('blockchain.claimtrie.search', args)
except RPCError as err:
error = err
finally:
await client.close()
finally:
print(json.dumps({
"time": loop.time() - start,
"error": error.__str__() if error else None,
"args": args,
}, indent=4))
if __name__ == "__main__":
asyncio.run(main())
-160
View File
@@ -1,160 +0,0 @@
import os
import sys
import json
import argparse
import asyncio
import time
import aiohttp
from aiohttp import ClientConnectorError
from lbry import __version__
from lbry.blob.blob_file import MAX_BLOB_SIZE
from lbry.conf import Config
from lbry.extras.daemon.client import daemon_rpc
from lbry.extras import system_info
async def report_to_slack(output, webhook):
payload = {
"text": f"lbrynet {__version__} ({system_info.get_platform()['platform']}) time to first byte:\n{output}"
}
async with aiohttp.request('post', webhook, data=json.dumps(payload)):
pass
def confidence(times, z, plus_err=True):
mean = sum(times) / len(times)
standard_dev = (sum(((t - sum(times) / len(times)) ** 2.0 for t in times)) / len(times)) ** 0.5
err = (z * standard_dev) / (len(times) ** 0.5)
return f"{round((mean + err) if plus_err else (mean - err), 3)}"
def variance(times):
mean = sum(times) / len(times)
return round(sum(((i - mean) ** 2.0 for i in times)) / (len(times) - 1), 3)
async def wait_for_done(conf, claim_name, timeout):
blobs_completed, last_completed = 0, time.perf_counter()
while True:
file = (await daemon_rpc(conf, "file_list", claim_name=claim_name))[0]
if file['status'] in ['finished', 'stopped']:
return True, file['blobs_completed'], file['blobs_in_stream']
elif blobs_completed < int(file['blobs_completed']):
blobs_completed, last_completed = int(file['blobs_completed']), time.perf_counter()
elif (time.perf_counter() - last_completed) > timeout:
return False, file['blobs_completed'], file['blobs_in_stream']
await asyncio.sleep(1.0)
async def main(cmd_args=None):
print('Time to first byte started using parameters:')
for key, value in vars(cmd_args).items():
print(f"{key}: {value}")
conf = Config()
url_to_claim = {}
try:
for page in range(1, cmd_args.download_pages + 1):
start = time.perf_counter()
kwargs = {
'page': page,
'claim_type': 'stream',
'order_by': ['trending_global', 'trending_mixed'],
'no_totals': True
}
# if not cmd_args.allow_fees:
# kwargs['fee_amount'] = 0
response = await daemon_rpc(
conf, 'claim_search', **kwargs
)
if 'error' in response or not response.get('items'):
print(f'Error getting claim list page {page}:')
print(response)
return 1
else:
url_to_claim.update({
claim['permanent_url']: claim for claim in response['items'] if claim['value_type'] == 'stream'
})
print(f'Claim search page {page} took: {time.perf_counter() - start}')
except (ClientConnectorError, ConnectionError):
print("Could not connect to daemon")
return 1
print("**********************************************")
print(f"Attempting to download {len(url_to_claim)} claim_search streams")
first_byte_times = []
download_speeds = []
download_successes = []
failed_to = {}
await asyncio.gather(*(
daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=claim['name'])
for claim in url_to_claim.values() if not cmd_args.keep_files
))
for i, (url, claim) in enumerate(url_to_claim.items()):
start = time.perf_counter()
response = await daemon_rpc(conf, 'get', uri=url, save_file=not cmd_args.head_blob_only)
if 'error' in response:
print(f"{i + 1}/{len(url_to_claim)} - failed to start {url}: {response['error']}")
failed_to[url] = 'start'
if cmd_args.exit_on_error:
return
continue
first_byte = time.perf_counter()
first_byte_times.append(first_byte - start)
print(f"{i + 1}/{len(url_to_claim)} - {first_byte - start} {url}")
if not cmd_args.head_blob_only:
downloaded, amount_downloaded, blobs_in_stream = await wait_for_done(
conf, claim['name'], cmd_args.stall_download_timeout
)
if downloaded:
download_successes.append(url)
else:
failed_to[url] = 'finish'
mbs = round((blobs_in_stream * (MAX_BLOB_SIZE - 1)) / (time.perf_counter() - start) / 1000000, 2)
download_speeds.append(mbs)
print(f"downloaded {amount_downloaded}/{blobs_in_stream} blobs for {url} at "
f"{mbs}mb/s")
if not cmd_args.keep_files:
await daemon_rpc(conf, 'file_delete', delete_from_download_dir=True, claim_name=claim['name'])
await asyncio.sleep(0.1)
print("**********************************************")
result = f"Started {len(first_byte_times)} of {len(url_to_claim)} attempted front page streams\n"
if first_byte_times:
result += f"Worst first byte time: {round(max(first_byte_times), 2)}\n" \
f"Best first byte time: {round(min(first_byte_times), 2)}\n" \
f"*95% confidence time-to-first-byte: {confidence(first_byte_times, 1.984)}s*\n" \
f"99% confidence time-to-first-byte: {confidence(first_byte_times, 2.626)}s\n" \
f"Variance: {variance(first_byte_times)}\n"
if download_successes:
result += f"Downloaded {len(download_successes)}/{len(url_to_claim)}\n" \
f"Best stream download speed: {round(max(download_speeds), 2)}mb/s\n" \
f"Worst stream download speed: {round(min(download_speeds), 2)}mb/s\n" \
f"95% confidence download speed: {confidence(download_speeds, 1.984, False)}mb/s\n" \
f"99% confidence download speed: {confidence(download_speeds, 2.626, False)}mb/s\n"
for reason in ('start', 'finish'):
failures = [url for url, why in failed_to.items() if reason == why]
if failures:
result += f"\nFailed to {reason}:\n" + "\n•".join(failures)
print(result)
webhook = os.environ.get('TTFB_SLACK_TOKEN', None)
if webhook:
await report_to_slack(result, webhook)
return 0
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--allow_fees", action='store_true')
parser.add_argument("--exit_on_error", action='store_true')
parser.add_argument("--stall_download_timeout", default=5, type=int)
parser.add_argument("--keep_files", action='store_true')
parser.add_argument("--head_blob_only", action='store_true')
parser.add_argument("--download_pages", type=int, default=10)
sys.exit(asyncio.run(main(cmd_args=parser.parse_args())) or 0)
-15
View File
@@ -1,15 +0,0 @@
#!/bin/bash
# entrypoint for wallet server Docker image
SNAPSHOT_URL="${SNAPSHOT_URL:-}" #off by default. latest snapshot at https://lbry.com/snapshot/wallet
if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
echo "Downloading wallet snapshot from $SNAPSHOT_URL"
wget -O wallet_snapshot.tar.bz2 "$SNAPSHOT_URL"
echo "Extracting snapshot..."
tar xvjf wallet_snapshot.tar.bz2 --directory /database
rm wallet_snapshot.tar.bz2
fi
/home/lbry/.local/bin/torba-server "$@"
-331
View File
@@ -1,331 +0,0 @@
import sys
import json
import random
import asyncio
import argparse
import traceback
from time import time
from datetime import datetime
try:
import aiohttp
import psycopg2
import slack
except ImportError:
print(f"To run {sys.argv[0]} you need to install aiohttp, psycopg2 and slackclient:")
print(f"")
print(f" $ pip install aiohttp psycopg2 slackclient")
print("")
sys.exit(1)
if not sys.version_info >= (3, 7):
print("Please use Python 3.7 or higher, this script expects that dictionary keys preserve order.")
sys.exit(1)
async def handle_slow_query(cursor, server, command, queries):
for query in queries:
cursor.execute("""
INSERT INTO wallet_server_slow_queries (server, command, query, event_time) VALUES (%s,%s,%s,%s);
""", (server, command, query, datetime.now()))
async def handle_analytics_event(cursor, event, server):
cursor.execute("""
INSERT INTO wallet_server_stats (server, sessions, event_time) VALUES (%s,%s,%s);
""", (server, event['status']['sessions'], datetime.now()))
for command, stats in event["api"].items():
data = {
'server': server,
'command': command,
'event_time': datetime.now()
}
for key, value in stats.items():
if key.endswith("_queries"):
if key == "interrupted_queries":
await handle_slow_query(cursor, server, command, value)
continue
if isinstance(value, list):
data.update({
key + '_avg': value[0],
key + '_min': value[1],
key + '_five': value[2],
key + '_twenty_five': value[3],
key + '_fifty': value[4],
key + '_seventy_five': value[5],
key + '_ninety_five': value[6],
key + '_max': value[7],
})
else:
data[key] = value
cursor.execute(f"""
INSERT INTO wallet_server_command_stats ({','.join(data)})
VALUES ({','.join('%s' for _ in data)});
""", list(data.values()))
SLACKCLIENT = None
async def boris_says(what_boris_says):
if SLACKCLIENT:
await SLACKCLIENT.chat_postMessage(
username="boris the wallet monitor",
icon_emoji=":boris:",
channel='#tech-sdk',
text=what_boris_says
)
else:
print(what_boris_says)
async def monitor(db, server):
c = db.cursor()
delay = 30
height_changed = None, time()
height_change_reported = False
first_attempt = True
while True:
try:
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session:
try:
ws = await session.ws_connect(server)
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
if first_attempt:
print(f"failed connecting to {server}")
await boris_says(random.choice([
f"{server} is not responding, probably dead, will not connect again.",
]))
return
raise
if first_attempt:
await boris_says(f"{server} is online")
else:
await boris_says(f"{server} is back online")
delay = 30
first_attempt = False
print(f"connected to {server}")
async for msg in ws:
event = json.loads(msg.data)
height = event['status']['height']
height_change_time = int(time()-height_changed[1])
if height_changed[0] != height:
height_changed = (height, time())
if height_change_reported:
await boris_says(
f"Server {server} received new block after {height_change_time / 60:.1f} minutes.",
)
height_change_reported = False
elif height_change_time > 10*60:
if not height_change_reported or height_change_time % (2*60) == 0:
await boris_says(
f"It's been {height_change_time/60:.1f} minutes since {server} received a new block.",
)
height_change_reported = True
await handle_analytics_event(c, event, server)
db.commit()
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
await boris_says(random.choice([
f"<!channel> Guys, we have a problem! Nobody home at {server}. Will check on it again in {delay} seconds.",
f"<!channel> Something wrong with {server}. I think dead. Will poke it again in {delay} seconds.",
f"<!channel> Don't hear anything from {server}, maybe dead. Will try it again in {delay} seconds.",
]))
await asyncio.sleep(delay)
delay += 30
async def main(dsn, servers):
db = ensure_database(dsn)
await boris_says(random.choice([
"No fear, Boris is here! I will monitor the servers now and will try not to fall asleep again.",
"Comrad the Cat and Boris are here now, monitoring wallet servers.",
]))
await asyncio.gather(*(
asyncio.create_task(monitor(db, server))
for server in servers
))
def ensure_database(dsn):
db = psycopg2.connect(**dsn)
c = db.cursor()
c.execute("SELECT to_regclass('wallet_server_stats');")
if c.fetchone()[0] is None:
print("creating table 'wallet_server_stats'...")
c.execute("""
CREATE TABLE wallet_server_stats (
server text,
sessions integer,
event_time timestamp
);
""")
c.execute("SELECT to_regclass('wallet_server_slow_queries');")
if c.fetchone()[0] is None:
print("creating table 'wallet_server_slow_queries'...")
c.execute("""
CREATE TABLE wallet_server_slow_queries (
server text,
command text,
query text,
event_time timestamp
);
""")
c.execute("SELECT to_regclass('wallet_server_command_stats');")
if c.fetchone()[0] is None:
print("creating table 'wallet_server_command_stats'...")
c.execute("""
CREATE TABLE wallet_server_command_stats (
server text,
command text,
event_time timestamp,
-- total requests received during event window
receive_count integer,
-- sum of these is total responses made
cache_response_count integer,
query_response_count integer,
intrp_response_count integer,
error_response_count integer,
-- millisecond timings for non-cache responses (response_*, interrupt_*, error_*)
response_avg float,
response_min float,
response_five float,
response_twenty_five float,
response_fifty float,
response_seventy_five float,
response_ninety_five float,
response_max float,
interrupt_avg float,
interrupt_min float,
interrupt_five float,
interrupt_twenty_five float,
interrupt_fifty float,
interrupt_seventy_five float,
interrupt_ninety_five float,
interrupt_max float,
error_avg float,
error_min float,
error_five float,
error_twenty_five float,
error_fifty float,
error_seventy_five float,
error_ninety_five float,
error_max float,
-- response, interrupt and error each also report the python, wait and sql stats
python_avg float,
python_min float,
python_five float,
python_twenty_five float,
python_fifty float,
python_seventy_five float,
python_ninety_five float,
python_max float,
wait_avg float,
wait_min float,
wait_five float,
wait_twenty_five float,
wait_fifty float,
wait_seventy_five float,
wait_ninety_five float,
wait_max float,
sql_avg float,
sql_min float,
sql_five float,
sql_twenty_five float,
sql_fifty float,
sql_seventy_five float,
sql_ninety_five float,
sql_max float,
-- extended timings for individual sql executions
individual_sql_avg float,
individual_sql_min float,
individual_sql_five float,
individual_sql_twenty_five float,
individual_sql_fifty float,
individual_sql_seventy_five float,
individual_sql_ninety_five float,
individual_sql_max float,
individual_sql_count integer
);
""")
db.commit()
return db
def get_dsn(args):
dsn = {}
for attr in ('dbname', 'user', 'password', 'host', 'port'):
value = getattr(args, f'pg_{attr}')
if value:
dsn[attr] = value
return dsn
def get_servers(args):
if '..' in args.server_range:
start, end = args.server_range.split('..')
else:
start = end = args.server_range
return [
args.server_url.format(i)
for i in range(int(start), int(end)+1)
]
def get_slack_client(args):
if args.slack_token:
return slack.WebClient(token=args.slack_token, run_async=True)
def get_args():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--pg-dbname", default="analytics", help="PostgreSQL database name")
parser.add_argument("--pg-user", help="PostgreSQL username")
parser.add_argument("--pg-password", help="PostgreSQL password")
parser.add_argument("--pg-host", default="localhost", help="PostgreSQL host")
parser.add_argument("--pg-port", default="5432", help="PostgreSQL port")
parser.add_argument("--server-url", default="http://spv{}.lbry.com:50005", help="URL with '{}' placeholder")
parser.add_argument("--server-range", default="1..5", help="Range of numbers or single number to use in URL placeholder")
parser.add_argument("--slack-token")
return parser.parse_args()
if __name__ == "__main__":
loop = asyncio.get_event_loop()
args = get_args()
SLACKCLIENT = get_slack_client(args)
try:
loop.run_until_complete(main(get_dsn(args), get_servers(args)))
except KeyboardInterrupt as e:
pass
except Exception as e:
loop.run_until_complete(boris_says("<!channel> I crashed with the following exception:"))
loop.run_until_complete(boris_says(traceback.format_exc()))
finally:
loop.run_until_complete(
boris_says(random.choice([
"Wallet servers will have to watch themselves, I'm leaving now.",
"I'm going to go take a nap, hopefully nothing blows up while I'm gone.",
"Babushka is calling, I'll be back later, someone else watch the servers while I'm gone.",
]))
)
@@ -1,179 +0,0 @@
import asyncio
import sqlite3
from lbry.testcase import CommandTestCase
from torba.client.basedatabase import SQLiteMixin
from lbry.wallet.dewies import dewies_to_lbc
from lbry.wallet.account import Account
def extract(d, keys):
return dict((k, d[k]) for k in keys)
class AccountManagement(CommandTestCase):
async def test_account_list_set_create_remove_add(self):
# check initial account
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 1)
# change account name and gap
account_id = response['lbc_regtest'][0]['id']
self.daemon.jsonrpc_account_set(
account_id=account_id, new_name='test account',
receiving_gap=95, receiving_max_uses=96,
change_gap=97, change_max_uses=98
)
response = (await self.daemon.jsonrpc_account_list())['lbc_regtest'][0]
self.assertEqual(response['name'], 'test account')
self.assertEqual(
response['address_generator']['receiving'],
{'gap': 95, 'maximum_uses_per_address': 96}
)
self.assertEqual(
response['address_generator']['change'],
{'gap': 97, 'maximum_uses_per_address': 98}
)
# create another account
await self.daemon.jsonrpc_account_create('second account')
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 2)
self.assertEqual(response['lbc_regtest'][1]['name'], 'second account')
account_id2 = response['lbc_regtest'][1]['id']
# make new account the default
self.daemon.jsonrpc_account_set(account_id=account_id2, default=True)
response = await self.daemon.jsonrpc_account_list(show_seed=True)
self.assertEqual(response['lbc_regtest'][0]['name'], 'second account')
account_seed = response['lbc_regtest'][1]['seed']
# remove account
self.daemon.jsonrpc_account_remove(response['lbc_regtest'][1]['id'])
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 1)
# add account
await self.daemon.jsonrpc_account_add('recreated account', seed=account_seed)
response = await self.daemon.jsonrpc_account_list()
self.assertEqual(len(response['lbc_regtest']), 2)
self.assertEqual(response['lbc_regtest'][1]['name'], 'recreated account')
# list specific account
response = await self.daemon.jsonrpc_account_list(account_id, include_claims=True)
self.assertEqual(response['name'], 'recreated account')
async def test_wallet_migration(self):
# null certificates should get deleted
await self.channel_create('@foo1')
await self.channel_create('@foo2')
await self.channel_create('@foo3')
keys = list(self.account.channel_keys.keys())
self.account.channel_keys[keys[0]] = None
self.account.channel_keys[keys[1]] = "some invalid junk"
await self.account.maybe_migrate_certificates()
self.assertEqual(list(self.account.channel_keys.keys()), [keys[2]])
async def assertFindsClaims(self, claim_names, awaitable):
self.assertEqual(claim_names, [txo.claim_name for txo in await awaitable])
async def assertOutputAmount(self, amounts, awaitable):
self.assertEqual(amounts, [dewies_to_lbc(txo.amount) for txo in await awaitable])
async def test_commands_across_accounts(self):
channel_list = self.daemon.jsonrpc_channel_list
stream_list = self.daemon.jsonrpc_stream_list
support_list = self.daemon.jsonrpc_support_list
utxo_list = self.daemon.jsonrpc_utxo_list
default_account = self.wallet.default_account
second_account = await self.daemon.jsonrpc_account_create('second account')
tx = await self.daemon.jsonrpc_account_send(
'0.05', await self.daemon.jsonrpc_address_unused(account_id=second_account.id)
)
await self.confirm_tx(tx.id)
await self.assertOutputAmount(['0.05', '9.949876'], utxo_list())
await self.assertOutputAmount(['0.05'], utxo_list(account_id=second_account.id))
await self.assertOutputAmount(['9.949876'], utxo_list(account_id=default_account.id))
channel1 = await self.channel_create('@channel-in-account1', '0.01')
channel2 = await self.channel_create(
'@channel-in-account2', '0.01', account_id=second_account.id, funding_account_ids=[default_account.id]
)
await self.assertFindsClaims(['@channel-in-account2', '@channel-in-account1'], channel_list())
await self.assertFindsClaims(['@channel-in-account1'], channel_list(account_id=default_account.id))
await self.assertFindsClaims(['@channel-in-account2'], channel_list(account_id=second_account.id))
stream1 = await self.stream_create('stream-in-account1', '0.01', channel_id=self.get_claim_id(channel1))
stream2 = await self.stream_create(
'stream-in-account2', '0.01', channel_id=self.get_claim_id(channel2),
account_id=second_account.id, funding_account_ids=[default_account.id]
)
await self.assertFindsClaims(['stream-in-account2', 'stream-in-account1'], stream_list())
await self.assertFindsClaims(['stream-in-account1'], stream_list(account_id=default_account.id))
await self.assertFindsClaims(['stream-in-account2'], stream_list(account_id=second_account.id))
await self.assertFindsClaims(
['stream-in-account2', 'stream-in-account1', '@channel-in-account2', '@channel-in-account1'],
self.daemon.jsonrpc_claim_list()
)
await self.assertFindsClaims(
['stream-in-account1', '@channel-in-account1'],
self.daemon.jsonrpc_claim_list(account_id=default_account.id)
)
await self.assertFindsClaims(
['stream-in-account2', '@channel-in-account2'],
self.daemon.jsonrpc_claim_list(account_id=second_account.id)
)
support1 = await self.support_create(self.get_claim_id(stream1), '0.01')
support2 = await self.support_create(
self.get_claim_id(stream2), '0.01', account_id=second_account.id, funding_account_ids=[default_account.id]
)
self.assertEqual([support2['txid'], support1['txid']], [txo.tx_ref.id for txo in await support_list()])
self.assertEqual([support1['txid']], [txo.tx_ref.id for txo in await support_list(account_id=default_account.id)])
self.assertEqual([support2['txid']], [txo.tx_ref.id for txo in await support_list(account_id=second_account.id)])
history = await self.daemon.jsonrpc_transaction_list()
self.assertEqual(len(history), 8)
self.assertEqual(extract(history[0]['support_info'][0], ['claim_name', 'is_tip', 'amount', 'balance_delta']), {
'claim_name': 'stream-in-account2',
'is_tip': False,
'amount': '0.01',
'balance_delta': '-0.01'
})
self.assertEqual(extract(history[1]['support_info'][0], ['claim_name', 'is_tip', 'amount', 'balance_delta']), {
'claim_name': 'stream-in-account1',
'is_tip': False,
'amount': '0.01',
'balance_delta': '-0.01'
})
self.assertEqual(extract(history[2]['claim_info'][0], ['claim_name', 'amount', 'balance_delta']), {
'claim_name': 'stream-in-account2',
'amount': '0.01',
'balance_delta': '-0.01'
})
self.assertEqual(extract(history[3]['claim_info'][0], ['claim_name', 'amount', 'balance_delta']), {
'claim_name': 'stream-in-account1',
'amount': '0.01',
'balance_delta': '-0.01'
})
self.assertEqual(extract(history[4]['claim_info'][0], ['claim_name', 'amount', 'balance_delta']), {
'claim_name': '@channel-in-account2',
'amount': '0.01',
'balance_delta': '-0.01'
})
self.assertEqual(extract(history[5]['claim_info'][0], ['claim_name', 'amount', 'balance_delta']), {
'claim_name': '@channel-in-account1',
'amount': '0.01',
'balance_delta': '-0.01'
})
self.assertEqual(history[6]['value'], '0.0')
self.assertEqual(history[7]['value'], '10.0')
async def test_address_validation(self):
address = await self.daemon.jsonrpc_address_unused()
bad_address = address[0:20] + '9999999' + address[27:]
with self.assertRaisesRegex(Exception, f"'{bad_address}' is not a valid address"):
await self.daemon.jsonrpc_account_send('0.1', addresses=[bad_address])
-65
View File
@@ -1,65 +0,0 @@
import contextlib
import asyncio
import logging
from io import StringIO
from torba.testcase import AsyncioTestCase
from lbry.conf import Config
from lbry.extras import cli
from lbry.extras.daemon.Components import (
DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
)
from lbry.extras.daemon.Daemon import Daemon
class CLIIntegrationTest(AsyncioTestCase):
async def asyncSetUp(self):
conf = Config()
conf.data_dir = '/tmp'
conf.share_usage_data = False
conf.api = 'localhost:5299'
conf.components_to_skip = (
DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
)
Daemon.component_attributes = {}
self.daemon = Daemon(conf)
await self.daemon.start()
async def asyncTearDown(self):
await self.daemon.stop(shutdown_runner=False)
def test_cli_status_command_with_auth(self):
actual_output = StringIO()
with contextlib.redirect_stdout(actual_output):
cli.main(["--api", "localhost:5299", "status"])
actual_output = actual_output.getvalue()
self.assertIn("connection_status", actual_output)
def test_setup_logging(self):
def setup(argv):
parser = cli.get_argument_parser()
args, command_args = parser.parse_known_args(argv)
loop = asyncio.get_event_loop()
conf = Config.create_from_arguments(args)
cli.setup_logging(args, conf, loop)
setup(["start"])
self.assertTrue(logging.getLogger("lbry").isEnabledFor(logging.INFO))
self.assertFalse(logging.getLogger("lbry").isEnabledFor(logging.DEBUG))
setup(["start", "--verbose"])
self.assertTrue(logging.getLogger("lbry").isEnabledFor(logging.DEBUG))
self.assertTrue(logging.getLogger("lbry").isEnabledFor(logging.INFO))
self.assertFalse(logging.getLogger("torba").isEnabledFor(logging.DEBUG))
setup(["start", "--verbose", "lbry.extras", "lbry.wallet", "torba.client"])
self.assertTrue(logging.getLogger("lbry.extras").isEnabledFor(logging.DEBUG))
self.assertTrue(logging.getLogger("lbry.wallet").isEnabledFor(logging.DEBUG))
self.assertTrue(logging.getLogger("torba.client").isEnabledFor(logging.DEBUG))
self.assertFalse(logging.getLogger("lbry").isEnabledFor(logging.DEBUG))
self.assertFalse(logging.getLogger("torba").isEnabledFor(logging.DEBUG))
@@ -1,401 +0,0 @@
import time
from math import ceil
from aiohttp import web
from lbry.testcase import CommandTestCase
class MockedCommentServer:
ERRORS = {
'INVALID_PARAMS': {'code': -32602, 'message': 'Invalid parameters'},
'INTERNAL': {'code': -32603, 'message': 'An internal error'},
'UNKNOWN': {'code': -1, 'message': 'An unknown or very miscellaneous error'},
'INVALID_METHOD': {'code': -32604, 'message': 'The Requested method does not exist'}
}
COMMENT_SCHEMA = {
'comment': None,
'comment_id': None,
'claim_id': None,
'parent_id': None,
'channel_name': None,
'channel_id': None,
'signature': None,
'signing_ts': None,
'timestamp': None,
'channel_url': None,
'is_hidden': False,
}
def __init__(self, port=2903):
self.port = port
self.app = web.Application(debug=True)
self.app.add_routes([web.post('/api', self.api)])
self.runner = None
self.server = None
self.comments = []
self.comment_id = 0
@classmethod
def _create_comment(cls, **kwargs):
schema = cls.COMMENT_SCHEMA.copy()
schema.update(**kwargs)
return schema
@staticmethod
def clean(d: dict):
return {k: v for k, v in d.items() if v or isinstance(v, bool)}
def create_comment(self, channel_name=None, channel_id=None, **kwargs):
comment_id = self.comment_id
channel_url = 'lbry://' + channel_name + '#' + channel_id if channel_id else None
comment = self._create_comment(
comment_id=str(comment_id),
channel_name=channel_name,
channel_id=channel_id,
channel_url=channel_url,
timestamp=str(int(time.time())),
**kwargs
)
self.comments.append(comment)
self.comment_id += 1
return self.clean(comment)
def abandon_comment(self, comment_id: int, channel_id: str, **kwargs):
deleted = False
try:
if 0 <= comment_id < len(self.comments) and self.comments[comment_id]['channel_id'] == channel_id:
self.comments.pop(comment_id)
deleted = True
finally:
return {
str(comment_id): {
'abandoned': deleted
}
}
def hide_comment(self, comment_id, signing_ts, signature):
comment_id = int(comment_id) if not isinstance(comment_id, int) else comment_id
if 0 <= comment_id < len(self.comments) and len(signature) == 128 and signing_ts.isalnum():
self.comments[comment_id]['is_hidden'] = True
return True
return False
def hide_comments(self, pieces: list):
comments_hidden = []
for p in pieces:
if self.hide_comment(**p):
comments_hidden.append(p['comment_id'])
return {'hidden': comments_hidden}
def get_claim_comments(self, claim_id, page=1, page_size=50,**kwargs):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
return {
'page': page,
'page_size': page_size,
'total_pages': ceil(len(comments)/page_size),
'total_items': len(comments),
'items': [self.clean(c) for c in (comments[::-1])[(page - 1) * page_size: page * page_size]],
'has_hidden_comments': bool(list(filter(lambda x: x['is_hidden'], comments)))
}
def get_claim_hidden_comments(self, claim_id, hidden=True, page=1, page_size=50):
comments = list(filter(lambda c: c['claim_id'] == claim_id, self.comments))
select_comments = list(filter(lambda c: c['is_hidden'] == hidden, comments))
return {
'page': page,
'page_size': page_size,
'total_pages': ceil(len(select_comments) / page_size),
'total_items': len(select_comments),
'items': [self.clean(c) for c in (select_comments[::-1])[(page - 1) * page_size: page * page_size]],
'has_hidden_comments': bool(list(filter(lambda c: c['is_hidden'], comments)))
}
def get_comment_channel_by_id(self, comment_id: int, **kwargs):
comment = self.comments[comment_id]
return {
'channel_id': comment.get('channel_id'),
'channel_name': comment.get('channel_name')
}
def get_comments_by_id(self, comment_ids: list):
comment_ids = [int(c) if not isinstance(c, int) else c for c in comment_ids]
comments = [self.comments[cmnt_id] for cmnt_id in comment_ids if 0 <= cmnt_id < len(self.comments)]
return comments
methods = {
'get_claim_comments': get_claim_comments,
'get_comments_by_id': get_comments_by_id,
'create_comment': create_comment,
'abandon_comment': abandon_comment,
'get_channel_from_comment_id': get_comment_channel_by_id,
'get_claim_hidden_comments': get_claim_hidden_comments,
'hide_comments': hide_comments,
}
def process_json(self, body) -> dict:
response = {'jsonrpc': '2.0', 'id': body['id']}
try:
if body['method'] in self.methods:
params: dict = body.get('params', {})
comment_id = params.get('comment_id')
if comment_id and not isinstance(comment_id, int):
params['comment_id'] = int(comment_id)
result = self.methods[body['method']](self, **params)
response['result'] = result
else:
response['error'] = self.ERRORS['INVALID_METHOD']
except Exception as err:
response['error'] = self.ERRORS['UNKNOWN']
response['error'].update({'exception': f'{type(err).__name__}: {err}'})
return response
async def start(self):
self.runner = web.AppRunner(self.app)
await self.runner.setup()
self.server = web.TCPSite(self.runner, 'localhost', self.port)
await self.server.start()
async def stop(self):
await self.runner.shutdown()
await self.runner.cleanup()
async def api(self, request):
body = await request.json()
if type(body) is list or type(body) is dict:
if type(body) is list:
response = [self.process_json(part) for part in body]
else:
response = self.process_json(body)
return web.json_response(response)
else:
raise TypeError('invalid type passed')
class CommentCommands(CommandTestCase):
async def asyncSetUp(self):
await super().asyncSetUp()
self.daemon.conf.comment_server = 'http://localhost:2903/api'
self.comment_server = MockedCommentServer(2903)
await self.comment_server.start()
self.addCleanup(self.comment_server.stop)
async def test01_comment_create(self):
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
stream = (await self.stream_create())['outputs'][0]
self.assertEqual(0, len((await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']))
comment = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
channel_id=channel['claim_id'],
comment="It's 5 O'Clock Somewhere"
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(1, len(comments))
self.assertEqual(comment['comment_id'], comments[0]['comment_id'])
self.assertEqual(stream['claim_id'], comments[0]['claim_id'])
channel2 = (await self.channel_create('@BuffettJimmy'))['outputs'][0]
await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
channel_name=channel2['name'],
comment='Let\'s all go to Margaritaville',
parent_id=comments[0]['comment_id']
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(2, len(comments))
self.assertEqual(comments[0]['channel_id'], channel2['claim_id'])
self.assertEqual(comments[0]['parent_id'], comments[1]['comment_id'])
comment = await self.daemon.jsonrpc_comment_create(
claim_id=stream['claim_id'],
comment='Anonymous comment'
)
comments = (await self.daemon.jsonrpc_comment_list(stream['claim_id']))['items']
self.assertEqual(comment['comment_id'], comments[0]['comment_id'])
async def test02_unsigned_comment_list(self):
stream = (await self.stream_create())['outputs'][0]
comments = []
num_items = 28
for i in range(num_items):
comment = await self.daemon.jsonrpc_comment_create(
comment=f'{i}',
claim_id=stream['claim_id'],
)
self.assertIn('comment_id', comment)
comments.append(comment)
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
comment_list = await self.daemon.jsonrpc_comment_list(stream['claim_id'])
for field in list_fields:
self.assertIn(field, comment_list)
self.assertEqual(comment_list['total_items'], num_items)
for comment in comment_list['items']:
self.assertEqual(comment['comment'], comments.pop()['comment'])
signed_comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id'],
is_channel_signature_valid=True
)
self.assertIs(len(signed_comment_list['items']), 0)
async def test03_signed_comments_list(self):
channel = (await self.channel_create('@JimmyBuffett'))['outputs'][0]
stream = (await self.stream_create())['outputs'][0]
comments = []
for i in range(28):
comment = await self.daemon.jsonrpc_comment_create(
comment=f'{i}',
claim_id=stream['claim_id'],
channel_id=channel['claim_id'],
)
self.assertIn('comment_id', comment)
comments.append(comment)
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id']
)
for field in list_fields:
self.assertIn(field, comment_list)
self.assertIs(comment_list['page_size'], 50)
self.assertIs(comment_list['page'], 1)
self.assertIs(comment_list['total_items'], 28)
for comment in comment_list['items']:
self.assertEqual(comment['comment'], comments.pop()['comment'])
signed_comment_list = await self.daemon.jsonrpc_comment_list(
claim_id=stream['claim_id'],
is_channel_signature_valid=True
)
self.assertIs(len(signed_comment_list['items']), 28)
async def test04_comment_abandons(self):
rswanson = (await self.channel_create('@RonSwanson'))['outputs'][0]
stream = (await self.stream_create('Pawnee Town Hall of Fame by Leslie Knope'))['outputs'][0]
comment = await self.daemon.jsonrpc_comment_create(
comment='KNOPE! WHAT DID I TELL YOU ABOUT PUTTING MY INFORMATION UP LIKE THAT',
claim_id=stream['claim_id'],
channel_id=rswanson['claim_id']
)
self.assertIn('signature', comment)
abandoned = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
self.assertIn(comment['comment_id'], abandoned)
self.assertTrue(abandoned[comment['comment_id']]['abandoned'])
abandoned = await self.daemon.jsonrpc_comment_abandon(comment['comment_id'])
self.assertFalse(abandoned[comment['comment_id']]['abandoned'])
async def test05_comment_hide(self):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
stream = await self.stream_create('Cool Lamps to Sit On', channel_id=moth_id)
claim_id = stream['outputs'][0]['claim_id']
comment1 = await self.daemon.jsonrpc_comment_create(
comment='Who on earth would want to sit around on a lamp all day',
claim_id=claim_id,
channel_id=bee['claim_id']
)
self.assertFalse(comment1['is_hidden'])
comment2 = await self.daemon.jsonrpc_comment_create(
comment='silence mortal',
claim_id=claim_id,
channel_id=moth_id,
)
self.assertFalse(comment2['is_hidden'])
comments = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertIn('has_hidden_comments', comments)
self.assertFalse(comments['has_hidden_comments'])
hidden = await self.daemon.jsonrpc_comment_hide([comment1['comment_id']])
self.assertIn('hidden', hidden)
hidden = hidden['hidden']
self.assertIn(comment1['comment_id'], hidden)
comments = await self.daemon.jsonrpc_comment_list(claim_id)
self.assertIn('has_hidden_comments', comments)
self.assertTrue(comments['has_hidden_comments'])
hidden_cmts1 = list(filter(lambda c: c['is_hidden'], comments['items']))
self.assertTrue(len(hidden_cmts1) == 1)
hidden_comment = hidden_cmts1[0]
self.assertEqual(hidden_comment['comment_id'], hidden[0])
hidden_comments = await self.daemon.jsonrpc_comment_list(claim_id, hidden=True)
self.assertIn('has_hidden_comments', hidden_comments)
self.assertTrue(hidden_comments['has_hidden_comments'])
self.assertLess(hidden_comments['total_items'], comments['total_items'])
self.assertListEqual(hidden_comments['items'], hidden_cmts1)
visible_comments = await self.daemon.jsonrpc_comment_list(claim_id, visible=True)
self.assertIn('has_hidden_comments', visible_comments)
self.assertTrue(visible_comments['has_hidden_comments'])
self.assertLess(visible_comments['total_items'], comments['total_items'])
total_hidden = hidden_comments['total_items']
total_visible = visible_comments['total_items']
self.assertEqual(total_hidden + total_visible, comments['total_items'])
items_hidden = hidden_comments['items']
items_visible = visible_comments['items']
for item in items_visible + items_hidden:
self.assertIn(item, comments['items'])
async def test06_comment_list_test(self):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
stream = await self.stream_create('Cool Lamps to Sit On', channel_id=moth_id)
claim_id = stream['outputs'][0]['claim_id']
hidden_comment = await self.daemon.jsonrpc_comment_create(
comment='Who on earth would want to sit around on a lamp all day',
claim_id=claim_id,
channel_id=bee['claim_id']
)
await self.daemon.jsonrpc_comment_hide([hidden_comment['comment_id']])
owner_comment = await self.daemon.jsonrpc_comment_create(
comment='Go away you yellow freak',
claim_id=claim_id,
channel_id=moth_id,
)
other_comment = await self.daemon.jsonrpc_comment_create(
comment='I got my swim trunks and my flippy-floppies',
claim_id=claim_id,
channel_id=bee['claim_id']
)
anon_comment = await self.daemon.jsonrpc_comment_create(
claim_id=claim_id,
comment='Anonymous comment'
)
all_comments = [anon_comment, other_comment, owner_comment, hidden_comment]
list_fields = ['items', 'page', 'page_size', 'has_hidden_comments', 'total_items', 'total_pages']
normal_list = await self.daemon.jsonrpc_comment_list(claim_id)
for field in list_fields:
self.assertIn(field, normal_list)
self.assertEqual(normal_list['total_items'], 4)
self.assertTrue(normal_list['has_hidden_comments'])
for i, cmnt in enumerate(all_comments):
self.assertEqual(cmnt['comment_id'], normal_list['items'][i]['comment_id'])
hidden = await self.daemon.jsonrpc_comment_list(claim_id, hidden=True)
self.assertTrue(hidden['has_hidden_comments'])
for field in list_fields:
self.assertIn(field, hidden)
self.assertEqual(hidden['total_items'], 1)
visible = await self.daemon.jsonrpc_comment_list(claim_id, visible=True)
for field in list_fields:
self.assertIn(field, visible)
self.assertTrue(visible['has_hidden_comments'])
self.assertEqual(visible['total_items'], normal_list['total_items'] - hidden['total_items'])
valid_list = await self.daemon.jsonrpc_comment_list(claim_id, is_channel_signature_valid=True)
for field in list_fields:
self.assertIn(field, valid_list)
self.assertTrue(visible['has_hidden_comments'])
self.assertEqual(len(valid_list['items']), len(normal_list['items']) - 1)
@@ -1,12 +0,0 @@
from lbry.testcase import CommandTestCase
class AddressManagement(CommandTestCase):
async def test_address_list(self):
addresses = await self.out(self.daemon.jsonrpc_address_list())
self.assertEqual(27, len(addresses))
single = await self.out(self.daemon.jsonrpc_address_list(addresses[11]['address']))
self.assertEqual(1, len(single))
self.assertEqual(single[0], addresses[11])
@@ -1,147 +0,0 @@
from lbry.testcase import CommandTestCase
class TransactionCommandsTestCase(CommandTestCase):
async def test_transaction_show(self):
# local tx
result = await self.out(self.daemon.jsonrpc_account_send(
'5.0', await self.daemon.jsonrpc_address_unused(self.account.id)
))
await self.confirm_tx(result['txid'])
tx = await self.daemon.jsonrpc_transaction_show(result['txid'])
self.assertEqual(tx.id, result['txid'])
# someone's tx
change_address = await self.blockchain.get_raw_change_address()
sendtxid = await self.blockchain.send_to_address(change_address, 10)
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
self.assertEqual(tx.id, sendtxid)
self.assertEqual(tx.height, -1)
await self.generate(1)
tx = await self.daemon.jsonrpc_transaction_show(sendtxid)
self.assertEqual(tx.height, self.ledger.headers.height)
# inexistent
result = await self.daemon.jsonrpc_transaction_show('0'*64)
self.assertFalse(result['success'])
async def test_utxo_release(self):
sendtxid = await self.blockchain.send_to_address(
await self.account.receiving.get_or_create_usable_address(), 1
)
await self.confirm_tx(sendtxid)
await self.assertBalance(self.account, '11.0')
await self.ledger.reserve_outputs(await self.account.get_utxos())
await self.assertBalance(self.account, '0.0')
await self.daemon.jsonrpc_utxo_release()
await self.assertBalance(self.account, '11.0')
async def test_granular_balances(self):
account2 = await self.daemon.jsonrpc_account_create("Tip-er")
account_balance = self.daemon.jsonrpc_account_balance
wallet_balance = self.daemon.jsonrpc_wallet_balance
expected = {
'total': '10.0',
'available': '10.0',
'reserved': '0.0',
'reserved_subtotals': None
}
self.assertEqual(await account_balance(reserved_subtotals=False), expected)
self.assertEqual(await wallet_balance(reserved_subtotals=False), expected)
expected = {
'total': '10.0',
'available': '10.0',
'reserved': '0.0',
'reserved_subtotals': {'claims': '0.0', 'supports': '0.0', 'tips': '0.0'}
}
self.assertEqual(await account_balance(reserved_subtotals=True), expected)
self.assertEqual(await wallet_balance(reserved_subtotals=True), expected)
# claim with update + supporting our own claim
stream1 = await self.stream_create('granularity', '3.0')
await self.stream_update(self.get_claim_id(stream1), data=b'news', bid='1.0')
await self.support_create(self.get_claim_id(stream1), '2.0')
expected = {
'total': '9.977534',
'available': '6.977534',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
}
self.assertEqual(await account_balance(reserved_subtotals=True), expected)
self.assertEqual(await wallet_balance(reserved_subtotals=True), expected)
address2 = await self.daemon.jsonrpc_address_unused(account2.id)
# send lbc to someone else
tx = await self.daemon.jsonrpc_account_send('1.0', address2)
await self.confirm_tx(tx.id)
self.assertEqual(await account_balance(reserved_subtotals=True), {
'total': '8.97741',
'available': '5.97741',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
'total': '9.97741',
'available': '6.97741',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
# tip received
support1 = await self.support_create(
self.get_claim_id(stream1), '0.3', tip=True, funding_account_ids=[account2.id]
)
self.assertEqual(await account_balance(reserved_subtotals=True), {
'total': '9.27741',
'available': '5.97741',
'reserved': '3.3',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'}
})
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
'total': '9.977268',
'available': '6.677268',
'reserved': '3.3',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.3'}
})
# tip claimed
tx = await self.daemon.jsonrpc_support_abandon(txid=support1['txid'], nout=0)
await self.confirm_tx(tx.id)
self.assertEqual(await account_balance(reserved_subtotals=True), {
'total': '9.277303',
'available': '6.277303',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
'total': '9.977161',
'available': '6.977161',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
stream2 = await self.stream_create(
'granularity-is-cool', '0.1', account_id=account2.id, funding_account_ids=[account2.id]
)
# tip another claim
await self.support_create(
self.get_claim_id(stream2), '0.2', tip=True, funding_account_ids=[self.account.id]
)
self.assertEqual(await account_balance(reserved_subtotals=True), {
'total': '9.077157',
'available': '6.077157',
'reserved': '3.0',
'reserved_subtotals': {'claims': '1.0', 'supports': '2.0', 'tips': '0.0'}
})
self.assertEqual(await wallet_balance(reserved_subtotals=True), {
'total': '9.938908',
'available': '6.638908',
'reserved': '3.3',
'reserved_subtotals': {'claims': '1.1', 'supports': '2.0', 'tips': '0.2'}
})
@@ -1,159 +0,0 @@
import asyncio
import json
from lbry.testcase import CommandTestCase
from torba.client.wallet import ENCRYPT_ON_DISK
class WalletEncryptionAndSynchronization(CommandTestCase):
SEED = (
"carbon smart garage balance margin twelve chest "
"sword toast envelope bottom stomach absent"
)
async def asyncSetUp(self):
await super().asyncSetUp()
self.daemon2 = await self.add_daemon(
seed="chest sword toast envelope bottom stomach absent "
"carbon smart garage balance margin twelve"
)
address = (await self.daemon2.wallet_manager.default_account.receiving.get_addresses(limit=1, only_usable=True))[0]
sendtxid = await self.blockchain.send_to_address(address, 1)
await self.confirm_tx(sendtxid, self.daemon2.ledger)
def assertWalletEncrypted(self, wallet_path, encrypted):
wallet = json.load(open(wallet_path))
self.assertEqual(wallet['accounts'][0]['private_key'][1:4] != 'prv', encrypted)
async def test_sync(self):
daemon, daemon2 = self.daemon, self.daemon2
# Preferences
self.assertFalse(daemon.jsonrpc_preference_get())
self.assertFalse(daemon2.jsonrpc_preference_get())
daemon.jsonrpc_preference_set("fruit", '["peach", "apricot"]')
daemon.jsonrpc_preference_set("one", "1")
daemon.jsonrpc_preference_set("conflict", "1")
daemon2.jsonrpc_preference_set("another", "A")
await asyncio.sleep(1)
# these preferences will win after merge since they are "newer"
daemon2.jsonrpc_preference_set("two", "2")
daemon2.jsonrpc_preference_set("conflict", "2")
daemon.jsonrpc_preference_set("another", "B")
self.assertDictEqual(daemon.jsonrpc_preference_get(), {
"one": "1", "conflict": "1", "another": "B", "fruit": ["peach", "apricot"]
})
self.assertDictEqual(daemon2.jsonrpc_preference_get(), {
"two": "2", "conflict": "2", "another": "A"
})
self.assertEqual(len((await daemon.jsonrpc_account_list())['lbc_regtest']), 1)
data = await daemon2.jsonrpc_sync_apply('password')
await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
self.assertEqual(len((await daemon.jsonrpc_account_list())['lbc_regtest']), 2)
self.assertDictEqual(
# "two" key added and "conflict" value changed to "2"
daemon.jsonrpc_preference_get(),
{"one": "1", "two": "2", "conflict": "2", "another": "B", "fruit": ["peach", "apricot"]}
)
# Channel Certificate
channel = await daemon2.jsonrpc_channel_create('@foo', '0.1')
await self.confirm_tx(channel.id, self.daemon2.ledger)
# both daemons will have the channel but only one has the cert so far
self.assertEqual(len(await daemon.jsonrpc_channel_list()), 1)
self.assertEqual(len(daemon.wallet_manager.default_wallet.accounts[1].channel_keys), 0)
self.assertEqual(len(await daemon2.jsonrpc_channel_list()), 1)
self.assertEqual(len(daemon2.wallet_manager.default_account.channel_keys), 1)
data = await daemon2.jsonrpc_sync_apply('password')
await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
# both daemons have the cert after sync'ing
self.assertEqual(
daemon2.wallet_manager.default_account.channel_keys,
daemon.wallet_manager.default_wallet.accounts[1].channel_keys
)
async def test_encryption_and_locking(self):
daemon = self.daemon
wallet = daemon.wallet_manager.default_wallet
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': False})
self.assertIsNone(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK))
self.assertWalletEncrypted(wallet.storage.path, False)
# can't lock an unencrypted account
with self.assertRaisesRegex(AssertionError, "Cannot lock an unencrypted wallet, encrypt first."):
daemon.jsonrpc_wallet_lock()
# safe to call unlock and decrypt, they are no-ops at this point
daemon.jsonrpc_wallet_unlock('password') # already unlocked
daemon.jsonrpc_wallet_decrypt() # already not encrypted
daemon.jsonrpc_wallet_encrypt('password')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet.storage.path, True)
daemon.jsonrpc_wallet_lock()
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': True, 'is_encrypted': True})
# can't sign transactions with locked wallet
with self.assertRaises(AssertionError):
await daemon.jsonrpc_channel_create('@foo', '1.0')
daemon.jsonrpc_wallet_unlock('password')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
await daemon.jsonrpc_channel_create('@foo', '1.0')
daemon.jsonrpc_wallet_decrypt()
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': False})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': False})
self.assertWalletEncrypted(wallet.storage.path, False)
async def test_sync_with_encryption_and_password_change(self):
daemon, daemon2 = self.daemon, self.daemon2
wallet, wallet2 = daemon.wallet_manager.default_wallet, daemon2.wallet_manager.default_wallet
self.assertEqual(wallet2.encryption_password, None)
self.assertEqual(wallet2.encryption_password, None)
daemon.jsonrpc_wallet_encrypt('password')
self.assertEqual(wallet.encryption_password, 'password')
data = await daemon2.jsonrpc_sync_apply('password2')
# sync_apply doesn't save password if encrypt-on-disk is False
self.assertEqual(wallet2.encryption_password, None)
# need to use new password2 in sync_apply
with self.assertRaises(ValueError): # wrong password
await daemon.jsonrpc_sync_apply('password', data=data['data'], blocking=True)
await daemon.jsonrpc_sync_apply('password2', data=data['data'], blocking=True)
# sync_apply with new password2 also sets it as new local password
self.assertEqual(wallet.encryption_password, 'password2')
self.assertEqual(daemon.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet.storage.path, True)
# check new password is active
daemon.jsonrpc_wallet_lock()
self.assertFalse(daemon.jsonrpc_wallet_unlock('password'))
self.assertTrue(daemon.jsonrpc_wallet_unlock('password2'))
# propagate disk encryption to daemon2
data = await daemon.jsonrpc_sync_apply('password3')
# sync_apply (even with no data) on wallet with encrypt-on-disk updates local password
self.assertEqual(wallet.encryption_password, 'password3')
self.assertEqual(wallet2.encryption_password, None)
await daemon2.jsonrpc_sync_apply('password3', data=data['data'], blocking=True)
# the other device got new password and on disk encryption
self.assertEqual(wallet2.encryption_password, 'password3')
self.assertEqual(daemon2.jsonrpc_wallet_status(), {'is_locked': False, 'is_encrypted': True})
self.assertEqual(daemon2.jsonrpc_preference_get(ENCRYPT_ON_DISK), {'encrypt-on-disk': True})
self.assertWalletEncrypted(wallet2.storage.path, True)
daemon2.jsonrpc_wallet_lock()
self.assertTrue(daemon2.jsonrpc_wallet_unlock('password3'))
@@ -1,61 +0,0 @@
import asyncio
import os
import lbry.wallet
from lbry.testcase import CommandTestCase
from lbry.extras.daemon.Components import HeadersComponent
from torba.client.basenetwork import ClientSession
from torba.testcase import IntegrationTestCase
class TestSessionBloat(IntegrationTestCase):
"""
Tests that server cleans up stale connections after session timeout and client times out too.
"""
LEDGER = lbry.wallet
async def test_session_bloat_from_socket_timeout(self):
await self.conductor.stop_spv()
await self.ledger.stop()
self.conductor.spv_node.session_timeout = 1
await self.conductor.start_spv()
session = ClientSession(
network=None, server=(self.conductor.spv_node.hostname, self.conductor.spv_node.port), timeout=0.2
)
await session.create_connection()
await session.send_request('server.banner', ())
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 1)
self.assertFalse(session.is_closing())
await asyncio.sleep(1.1)
with self.assertRaises(asyncio.TimeoutError):
await session.send_request('server.banner', ())
self.assertTrue(session.is_closing())
self.assertEqual(len(self.conductor.spv_node.server.session_mgr.sessions), 0)
class TestSegwitServer(IntegrationTestCase):
LEDGER = lbry.wallet
ENABLE_SEGWIT = True
async def test_at_least_it_starts(self):
await asyncio.wait_for(self.ledger.network.get_headers(0, 1), 1.0)
class TestHeadersComponent(CommandTestCase):
LEDGER = lbry.wallet
async def asyncSetUp(self):
await super().asyncSetUp()
self.component_manager = self.daemon.component_manager
self.component_manager.conf.blockchain_name = 'lbrycrd_main'
self.headers_component = HeadersComponent(self.component_manager)
async def test_cant_reach_host(self):
HeadersComponent.HEADERS_URL = 'notthere/'
os.unlink(self.headers_component.headers.path)
# test is that this doesn't raise
await self.headers_component.start()
self.assertTrue(self.component_manager.get_components_status()['blockchain_headers'])
self.assertEqual(await self.headers_component.get_status(), {})
@@ -1,59 +0,0 @@
from torba.testcase import AsyncioTestCase
import hashlib
from lbry.extras.daemon.comment_client import sign_comment
from lbry.extras.daemon.comment_client import is_comment_signed_by_channel
from tests.unit.wallet.test_schema_signing import get_stream, get_channel
class TestSigningComments(AsyncioTestCase):
@staticmethod
def create_claim_comment_body(comment, claim, channel):
return {
'claim_id': claim.claim_id,
'channel_name': channel.claim_name,
'channel_id': channel.claim_id,
'comment': comment,
'comment_id': hashlib.sha256(comment.encode()).hexdigest()
}
def test01_successful_create_sign_and_validate_comment(self):
channel = get_channel('@BusterBluth')
stream = get_stream('pop secret')
comment = self.create_claim_comment_body('Cool stream', stream, channel)
sign_comment(comment, channel)
self.assertTrue(is_comment_signed_by_channel(comment, channel))
def test02_fail_to_validate_spoofed_channel(self):
pdiddy = get_channel('@PDitty')
channel2 = get_channel('@TomHaverford')
stream = get_stream()
comment = self.create_claim_comment_body('Woahh This is Sick!! Shout out 2 my boy Tommy H', stream, pdiddy)
sign_comment(comment, channel2)
self.assertFalse(is_comment_signed_by_channel(comment, pdiddy))
def test03_successful_sign_abandon_comment(self):
rswanson = get_channel('@RonSwanson')
dsilver = get_stream('Welcome to the Pawnee, and give a big round for Ron Swanson, AKA Duke Silver')
comment_body = self.create_claim_comment_body('COMPUTER, DELETE ALL VIDEOS OF RON.', dsilver, rswanson)
sign_comment(comment_body, rswanson, abandon=True)
self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, abandon=True))
def test04_invalid_signature(self):
rswanson = get_channel('@RonSwanson')
jeanralphio = get_channel('@JeanRalphio')
chair = get_stream('This is a nice chair. I made it with Mahogany wood and this electric saw')
chair_comment = self.create_claim_comment_body(
'Hah. You use an electric saw? Us swansons have been making chairs with handsaws just three after birth.',
chair,
rswanson
)
sign_comment(chair_comment, rswanson)
self.assertTrue(is_comment_signed_by_channel(chair_comment, rswanson))
self.assertFalse(is_comment_signed_by_channel(chair_comment, jeanralphio))
fake_abandon_signal = chair_comment.copy()
sign_comment(fake_abandon_signal, jeanralphio, abandon=True)
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, rswanson, abandon=True))
self.assertFalse(is_comment_signed_by_channel(fake_abandon_signal, jeanralphio, abandon=True))
-23
View File
@@ -1,23 +0,0 @@
import unittest
from lbry.schema.tags import normalize_tag, clean_tags
class TestTagNormalization(unittest.TestCase):
def assertNormalizedTag(self, clean, dirty):
self.assertEqual(clean, normalize_tag(dirty))
def test_normalize_tag(self):
tag = self.assertNormalizedTag
tag('', ' \t #!~')
tag('tag', 'T\'ag')
tag('t ag', '\tT \nAG ')
tag('tag hash', '#tag~#hash!')
def test_clean_tags(self):
self.assertEqual(['tag'], clean_tags([' \t #!~', '!taG', '\t']))
cleaned = clean_tags(['fOo', '!taG', 'FoO'])
self.assertIn('tag', cleaned)
self.assertIn('foo', cleaned)
self.assertEqual(len(cleaned), 2)
View File
Whitespace-only changes.
Whitespace-only changes.
@@ -1,60 +0,0 @@
import time
import unittest
from lbry.wallet.server.metrics import ServerLoadData, calculate_avg_percentiles
class TestPercentileCalculation(unittest.TestCase):
def test_calculate_percentiles(self):
self.assertEqual(calculate_avg_percentiles([]), (0, 0, 0, 0, 0, 0, 0, 0))
self.assertEqual(calculate_avg_percentiles([1]), (1, 1, 1, 1, 1, 1, 1, 1))
self.assertEqual(calculate_avg_percentiles([1, 2]), (1, 1, 1, 1, 1, 2, 2, 2))
self.assertEqual(calculate_avg_percentiles([1, 2, 3]), (2, 1, 1, 1, 2, 3, 3, 3))
self.assertEqual(calculate_avg_percentiles([4, 1, 2, 3]), (2, 1, 1, 1, 2, 3, 4, 4))
self.assertEqual(calculate_avg_percentiles([1, 2, 3, 4, 5, 6]), (3, 1, 1, 2, 3, 5, 6, 6))
self.assertEqual(calculate_avg_percentiles(
list(range(1, 101))), (50, 1, 5, 25, 50, 75, 95, 100))
class TestCollectingMetrics(unittest.TestCase):
def test_happy_path(self):
self.maxDiff = None
load = ServerLoadData()
search = load.for_api('search')
self.assertEqual(search.name, 'search')
search.start()
search.cache_response()
search.cache_response()
metrics = {
'search': [{'total': 40}],
'execute_query': [
{'total': 20},
{'total': 10}
]
}
for x in range(5):
search.query_response(time.perf_counter() - 0.055 + 0.001*x, metrics)
metrics['execute_query'][0]['total'] = 10
metrics['execute_query'][0]['sql'] = "select lots, of, stuff FROM claim where something=1"
search.query_interrupt(time.perf_counter() - 0.050, metrics)
search.query_error(time.perf_counter() - 0.050, metrics)
search.query_error(time.perf_counter() - 0.052, {})
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {'search': {
"receive_count": 1,
"cache_response_count": 2,
"query_response_count": 5,
"intrp_response_count": 1,
"error_response_count": 2,
"response": (53, 51, 51, 52, 53, 54, 55, 55),
"interrupt": (50, 50, 50, 50, 50, 50, 50, 50),
"error": (51, 50, 50, 50, 50, 52, 52, 52),
"python": (12, 10, 10, 10, 10, 20, 20, 20),
"wait": (12, 10, 10, 10, 12, 14, 15, 15),
"sql": (27, 20, 20, 20, 30, 30, 30, 30),
"individual_sql": (13, 10, 10, 10, 10, 20, 20, 20),
"individual_sql_count": 14,
"errored_queries": ['FROM claim where something=1'],
"interrupted_queries": ['FROM claim where something=1'],
}}})
self.assertEqual(load.to_json_and_reset({}), {'status': {}, 'api': {}})
+4
View File
@@ -0,0 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.38.0rc6"
version = tuple(__version__.split('.'))
@@ -1,4 +1,4 @@
from lbry.cryptoutils import get_lbry_hash_obj
from lbrynet.cryptoutils import get_lbry_hash_obj
MAX_BLOB_SIZE = 2 * 2 ** 20
@@ -10,12 +10,12 @@ from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
from lbry.cryptoutils import backend, get_lbry_hash_obj
from lbry.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
from lbrynet.cryptoutils import backend, get_lbry_hash_obj
from lbrynet.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
from lbry.blob import MAX_BLOB_SIZE, blobhash_length
from lbry.blob.blob_info import BlobInfo
from lbry.blob.writer import HashBlobWriter
from lbrynet.blob import MAX_BLOB_SIZE, blobhash_length
from lbrynet.blob.blob_info import BlobInfo
from lbrynet.blob.writer import HashBlobWriter
log = logging.getLogger(__name__)
@@ -71,7 +71,7 @@ class AbstractBlob:
'readers'
]
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
self.loop = loop
@@ -175,7 +175,7 @@ class AbstractBlob:
@classmethod
async def create_from_unencrypted(
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
"""
@@ -237,7 +237,7 @@ class BlobBuffer(AbstractBlob):
"""
An in-memory only blob
"""
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
self._verified_bytes: typing.Optional[BytesIO] = None
@@ -276,7 +276,7 @@ class BlobFile(AbstractBlob):
"""
A blob existing on the local file system
"""
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
@@ -324,7 +324,7 @@ class BlobFile(AbstractBlob):
@classmethod
async def create_from_unencrypted(
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
asyncio.Task]] = None) -> BlobInfo:
File renamed without changes.
@@ -2,21 +2,21 @@ import os
import typing
import asyncio
import logging
from lbry.utils import LRUCache
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbry.stream.descriptor import StreamDescriptor
from lbry.connection_manager import ConnectionManager
from lbrynet.utils import LRUCache
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.connection_manager import ConnectionManager
if typing.TYPE_CHECKING:
from lbry.conf import Config
from lbry.dht.protocol.data_store import DictDataStore
from lbry.extras.daemon.storage import SQLiteStorage
from lbrynet.conf import Config
from lbrynet.dht.protocol.data_store import DictDataStore
from lbrynet.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
class BlobManager:
def __init__(self, loop: asyncio.AbstractEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
node_data_store: typing.Optional['DictDataStore'] = None):
"""
This class stores blobs on the hard disk
@@ -37,14 +37,18 @@ class BlobManager:
self.connection_manager = ConnectionManager(loop)
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
if self.config.save_blobs or (
is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))):
if self.config.save_blobs:
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
return BlobBuffer(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
else:
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
return BlobBuffer(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
if blob_hash in self.blobs:
@@ -65,10 +69,10 @@ class BlobManager:
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
if not is_valid_blobhash(blob_hash):
raise ValueError(blob_hash)
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return False
if blob_hash in self.blobs:
return self.blobs[blob_hash].get_is_verified()
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return False
return self._get_blob(blob_hash, length).get_is_verified()
async def setup(self) -> bool:
@@ -78,13 +82,11 @@ class BlobManager:
return {
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
}
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
if to_add:
self.completed_blob_hashes.update(to_add)
if self.config.track_bandwidth:
self.connection_manager.start()
self.connection_manager.start()
return True
def stop(self):
@@ -2,8 +2,8 @@ import typing
import logging
import asyncio
from io import BytesIO
from lbry.error import InvalidBlobHashError, InvalidDataError
from lbry.cryptoutils import get_lbry_hash_obj
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from lbrynet.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
File renamed without changes.
Loaded 100 of 397 files, more files were not shown because too many files have changed in this diff. Show more