Compare commits
111
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
03dff4b382 | ||
|
|
f546b20a01 | ||
|
|
0e8d9a264f | ||
|
|
66a4c98bee | ||
|
|
36101db500 | ||
|
|
6bff298d1e | ||
|
|
c832f8ffbb | ||
|
|
d7fe46dbde | ||
|
|
5951186463 | ||
|
|
7191042bb8 | ||
|
|
a80fbcc252 | ||
|
|
c321758afd | ||
|
|
ca8f59a643 | ||
|
|
8110882617 | ||
|
|
a283b66541 | ||
|
|
569de37e16 | ||
|
|
669f3394c7 | ||
|
|
87ec2c7378 | ||
|
|
ea7a31cad1 | ||
|
|
c894ade25e | ||
|
|
a3fe127a92 | ||
|
|
7f526be879 | ||
|
|
71ae174e29 | ||
|
|
2277d134cc | ||
|
|
880aa265f1 | ||
|
|
ea639e64a4 | ||
|
|
b60fdee5e2 | ||
|
|
44f07fb81d | ||
|
|
60a85fadd3 | ||
|
|
68367125e1 | ||
|
|
6112f60681 | ||
|
|
ecc74e2ae5 | ||
|
|
c2dae43965 | ||
|
|
6e3a25d255 | ||
|
|
4e13cf1d85 | ||
|
|
0fee1897eb | ||
|
|
9dd51467fe | ||
|
|
cec6dca72f | ||
|
|
58df2b9bcf | ||
|
|
cc6bdd6295 | ||
|
|
7a044f9f78 | ||
|
|
e77acde005 | ||
|
|
ece5082096 | ||
|
|
0723026a56 | ||
|
|
0b7346f4a1 | ||
|
|
a357db1ff6 | ||
|
|
a2287f8d0a | ||
|
|
076decd6e9 | ||
|
|
d40461de94 | ||
|
|
4b1492be0d | ||
|
|
1676246743 | ||
|
|
f7c9f6d142 | ||
|
|
6022339091 | ||
|
|
25d97afa60 | ||
|
|
df635b963c | ||
|
|
6d0a054170 | ||
|
|
177d3af77f | ||
|
|
35f71c619b | ||
|
|
31a80ccce4 | ||
|
|
3194cec8df | ||
|
|
57fd3c5801 | ||
|
|
86617f1bda | ||
|
|
35f36c598c | ||
|
|
36bd7816fe | ||
|
|
fd632392d4 | ||
|
|
423b48866f | ||
|
|
745bc24343 | ||
|
|
04a823c7d0 | ||
|
|
56af4c2fcb | ||
|
|
77daf6812a | ||
|
|
c7e964ec42 | ||
|
|
694e2c2a4f | ||
|
|
df6537fae8 | ||
|
|
71f6542571 | ||
|
|
5a6294c08e | ||
|
|
f0390786d6 | ||
|
|
719d18c670 | ||
|
|
a116dcd3bb | ||
|
|
e2dd3dcf88 | ||
|
|
822f0c3cb2 | ||
|
|
63fb39016b | ||
|
|
e79f8d58ca | ||
|
|
a469dfb583 | ||
|
|
6cbc545f84 | ||
|
|
ea1e24d8f9 | ||
|
|
6fcb4d93c9 | ||
|
|
36031c9d0f | ||
|
|
a70f5e2440 | ||
|
|
6c42aab567 | ||
|
|
27631a1746 | ||
|
|
320f5cebc7 | ||
|
|
ea2a583803 | ||
|
|
9965801258 | ||
|
|
24ed0521c7 | ||
|
|
d8fed79810 | ||
|
|
c30db15efa | ||
|
|
817869b915 | ||
|
|
3038dd9440 | ||
|
|
276b2c737a | ||
|
|
62543b94c9 | ||
|
|
4d607ed276 | ||
|
|
26e2865e35 | ||
|
|
f7c94c9eb5 | ||
|
|
25a91f89fd | ||
|
|
246f055145 | ||
|
|
8c3bf0d4cb | ||
|
|
e49f24e95f | ||
|
|
48a18ff771 | ||
|
|
37161057c6 | ||
|
|
68d1157495 | ||
|
|
19c3b2f4b6 |
No files matched your search
+179
@@ -0,0 +1,179 @@
|
||||
default:
|
||||
image: python:3.7
|
||||
|
||||
|
||||
#cache:
|
||||
# directories:
|
||||
# - $HOME/venv
|
||||
# - $HOME/.cache/pip
|
||||
# - $HOME/Library/Caches/pip
|
||||
# - $HOME/Library/Caches/Homebrew
|
||||
# - $TRAVIS_BUILD_DIR/.tox
|
||||
|
||||
|
||||
stages:
|
||||
- test
|
||||
- build
|
||||
- release
|
||||
|
||||
|
||||
test:lint:
|
||||
stage: test
|
||||
script:
|
||||
- make install
|
||||
- make lint
|
||||
|
||||
test:lbry-unit:
|
||||
stage: test
|
||||
script:
|
||||
- make install
|
||||
- cd lbry && HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
|
||||
test:lbry-integ:
|
||||
stage: test
|
||||
script:
|
||||
- pip install coverage tox-travis
|
||||
- cd lbry && tox
|
||||
|
||||
|
||||
.torba-tests: &torba_tests
|
||||
stage: test
|
||||
script:
|
||||
- pip install coverage tox-travis
|
||||
- cd torba && tox
|
||||
|
||||
test:torba-unit:
|
||||
before_script:
|
||||
- export TESTTYPE=unit
|
||||
<<: *torba_tests
|
||||
|
||||
test:torba-integ:
|
||||
before_script:
|
||||
- export TESTTYPE=integration
|
||||
<<: *torba_tests
|
||||
|
||||
test:json-api:
|
||||
stage: test
|
||||
script:
|
||||
- make install
|
||||
- cd lbry && HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
|
||||
|
||||
|
||||
.build:
|
||||
stage: build
|
||||
artifacts:
|
||||
expire_in: 1 day
|
||||
paths:
|
||||
- lbrynet-${OS}.zip
|
||||
script:
|
||||
- pip install pyinstaller
|
||||
- pip install -e torba/.
|
||||
- python3.7 docker/set_build.py
|
||||
- pip install -e lbry/.
|
||||
- pyinstaller --onefile --name lbrynet lbry/lbry/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
|
||||
- openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
|
||||
- dist/lbrynet --version
|
||||
|
||||
build:linux:
|
||||
extends: .build
|
||||
image: ubuntu:18.04 # cant use python3.7 image - binary won't run on ubuntu
|
||||
variables:
|
||||
OS: linux
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
||||
- apt-get install -y --no-install-recommends python3.7-dev python3-setuptools python3-wheel
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
|
||||
build:mac:
|
||||
extends: .build
|
||||
tags: [macos] # makes gitlab use the mac runner
|
||||
variables:
|
||||
OS: mac
|
||||
GIT_DEPTH: 5
|
||||
VENV: /tmp/gitlab-lbry-sdk-venv
|
||||
before_script:
|
||||
# - brew upgrade python || true
|
||||
- python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
|
||||
# - pip3 install --user --upgrade pip virtualenv
|
||||
- pip3 --version | grep -q '\(python 3\.7\)'
|
||||
- virtualenv --python=python3.7 "${VENV}"
|
||||
- source "${VENV}/bin/activate"
|
||||
after_script:
|
||||
- rm -rf "${VENV}"
|
||||
|
||||
build:windows:
|
||||
extends: .build
|
||||
tags: [windows] # makes gitlab use the windows runner
|
||||
variables:
|
||||
OS: windows
|
||||
GIT_DEPTH: 5
|
||||
before_script:
|
||||
- ./docker/install_choco.ps1
|
||||
- choco install -y --x86 python3 7zip checksum
|
||||
# - python --version | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
|
||||
# - pip --version | findstr /E '\(python 3\.7\)'
|
||||
- pip install virtualenv pywin32
|
||||
- virtualenv venv
|
||||
- venv/Scripts/activate.ps1
|
||||
after_script:
|
||||
- rmdir -Recurse venv
|
||||
script:
|
||||
- pip install pyinstaller
|
||||
- pip install -e torba/.
|
||||
- python docker/set_build.py
|
||||
- pip install -e lbry/.
|
||||
- pyinstaller --additional-hooks-dir=lbry/scripts/. --icon=lbry/icons/lbry256.ico -F -n lbrynet lbry/lbry/extras/cli.py
|
||||
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip dist/lbrynet.exe
|
||||
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
|
||||
- dist/lbrynet.exe --version
|
||||
|
||||
|
||||
# upload could be done by making it a yaml alias and putting it right into the build step. that way if one OS fails, the others still get uploaded
|
||||
.upload:
|
||||
stage: release
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- pip install awscli
|
||||
- S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
|
||||
- AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
|
||||
aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
|
||||
|
||||
s3:linux:
|
||||
extends: .upload
|
||||
variables:
|
||||
OS: linux
|
||||
dependencies:
|
||||
- build:linux
|
||||
|
||||
s3:mac:
|
||||
extends: .upload
|
||||
variables:
|
||||
OS: mac
|
||||
dependencies:
|
||||
- build:mac
|
||||
|
||||
s3:windows:
|
||||
extends: .upload
|
||||
variables:
|
||||
OS: windows
|
||||
dependencies:
|
||||
- build:windows
|
||||
|
||||
#release:linux:
|
||||
# stage: release
|
||||
# only: [tags]
|
||||
# variables:
|
||||
# OS: linux
|
||||
# GIT_STRATEGY: none
|
||||
# dependencies: [build:linux]
|
||||
# script:
|
||||
# - pip install githubrelease
|
||||
# - githubrelease asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
|
||||
|
||||
+24
-16
@@ -13,22 +13,26 @@ jobs:
|
||||
|
||||
- stage: test
|
||||
name: "LBRY Unit Tests"
|
||||
before_install:
|
||||
- sudo add-apt-repository -y ppa:jonathonf/backports
|
||||
- sudo apt-get update && sudo apt-get -y install sqlite3
|
||||
install:
|
||||
- make install
|
||||
script:
|
||||
- cd lbry && HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
after_success:
|
||||
- coverage combine lbry/
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- name: "LBRY Integration Tests"
|
||||
before_install:
|
||||
- sudo add-apt-repository -y ppa:jonathonf/backports
|
||||
- sudo apt-get update && sudo apt-get -y install sqlite3
|
||||
install:
|
||||
- pip install coverage tox-travis
|
||||
- sudo mount -o mode=1777,nosuid,nodev -t tmpfs tmpfs /tmp
|
||||
script: cd lbry && tox
|
||||
after_success:
|
||||
- coverage combine lbry
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- &torba-tests
|
||||
name: "Torba Unit Tests"
|
||||
@@ -38,38 +42,40 @@ jobs:
|
||||
script: cd torba && tox
|
||||
after_success:
|
||||
- coverage combine torba/tests
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- <<: *torba-tests
|
||||
name: "Torba Integration Tests"
|
||||
env: TESTTYPE=integration
|
||||
|
||||
- name: "Run Examples"
|
||||
before_install:
|
||||
- sudo add-apt-repository -y ppa:jonathonf/backports
|
||||
- sudo apt-get update && sudo apt-get -y install sqlite3
|
||||
install:
|
||||
- make install
|
||||
script:
|
||||
- cd lbry && HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
after_success:
|
||||
- coverage combine lbry
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
- &build
|
||||
stage: build
|
||||
name: "Linux"
|
||||
env: OS=linux
|
||||
before_install:
|
||||
- sudo add-apt-repository -y ppa:jonathonf/backports
|
||||
- sudo apt-get update && sudo apt-get -y install sqlite3
|
||||
install:
|
||||
- pip install pyinstaller awscli
|
||||
- cd torba && pip install -e . && cd ..
|
||||
- cd lbry
|
||||
- python scripts/set_build.py
|
||||
- pip install -e .
|
||||
- python docker/set_build.py
|
||||
- pip install -e torba/.
|
||||
- pip install -e lbry/.
|
||||
script:
|
||||
- pyinstaller -F -n lbrynet lbry/extras/cli.py
|
||||
- cd dist
|
||||
- chmod +x lbrynet
|
||||
- zip -j lbrynet-${OS}.zip lbrynet
|
||||
- pyinstaller -F -n lbrynet lbry/lbry/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip --junk-paths lbrynet-${OS}.zip dist/lbrynet
|
||||
- shasum -a 256 -b lbrynet-${OS}.zip
|
||||
- ./lbrynet --version
|
||||
- dist/lbrynet --version
|
||||
after_success:
|
||||
- aws configure set aws_access_key_id $ARTIFACTS_KEY
|
||||
- aws configure set aws_secret_access_key $ARTIFACTS_SECRET
|
||||
@@ -114,19 +120,21 @@ jobs:
|
||||
- python -m pip install --upgrade pip
|
||||
- pip install pywin32
|
||||
script:
|
||||
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
|
||||
- pyinstaller --additional-hooks-dir=lbry/scripts/. --icon=lbry/icons/lbry256.ico -F -n lbrynet lbry/lbry/extras/cli.py
|
||||
- cd dist
|
||||
- 7z a -tzip lbrynet-windows.zip lbrynet.exe
|
||||
- sha256sum -b lbrynet-windows.zip
|
||||
- ./lbrynet.exe --version
|
||||
|
||||
- if: tag IS present
|
||||
stage: build
|
||||
name: "Wallet Server Docker Image - Tagged Release"
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f lbry/scripts/Dockerfile.wallet_server .
|
||||
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -t lbry/wallet-server:latest-release -f docker/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:$TRAVIS_TAG
|
||||
- docker push lbry/wallet-server:latest-release
|
||||
|
||||
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
|
||||
stage: build
|
||||
@@ -134,7 +142,7 @@ jobs:
|
||||
script:
|
||||
- set -e
|
||||
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f lbry/scripts/Dockerfile.wallet_server .
|
||||
- travis_retry docker build -t lbry/wallet-server:master -f docker/Dockerfile.wallet_server .
|
||||
- docker push lbry/wallet-server:master
|
||||
|
||||
cache:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry-sdk) [](https://codecov.io/gh/lbryio/lbry-sdk)
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry-sdk)
|
||||
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
|
||||
@@ -8,7 +8,9 @@ RUN mkdir -p /home/$user/
|
||||
RUN chown -R $user:$user /home/$user/
|
||||
|
||||
# install python, pip, git and clean up
|
||||
RUN apt-get update && apt-get -y --no-install-recommends install build-essential git tar wget python3.7 python3.7-dev python3-pip && rm -rf /var/lib/apt/lists/*
|
||||
RUN apt-get update && \
|
||||
apt-get -y --no-install-recommends install build-essential git tar wget python3.7 python3.7-dev python3-pip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# create and chown database dir
|
||||
ARG db_dir=/database
|
||||
@@ -26,8 +28,7 @@ RUN python3.7 -m pip install --upgrade pip setuptools
|
||||
RUN python3.7 -m pip install --user uvloop
|
||||
|
||||
# copy lbrynet
|
||||
ARG projects_dir=/home/$user/projects
|
||||
RUN mkdir $projects_dir
|
||||
ARG projects_dir=/home/$user
|
||||
COPY . $projects_dir
|
||||
USER root
|
||||
RUN chown -R $user:$user .
|
||||
@@ -35,7 +36,7 @@ USER $user
|
||||
|
||||
# install torba & lbry
|
||||
WORKDIR $projects_dir/torba
|
||||
RUN python3.7 -m pip install --user .
|
||||
RUN python3.7 -m pip install --user -e .
|
||||
WORKDIR $projects_dir/lbry
|
||||
RUN python3.7 -m pip install --user -e .
|
||||
RUN rm ~/.cache -rf
|
||||
@@ -53,5 +54,5 @@ ENV BANDWIDTH_LIMIT=1000000000000000000000000000000000000000000
|
||||
ENV MAX_SESSIONS=1000000000
|
||||
ENV MAX_SEND=1000000000000000000
|
||||
ENV EVENT_LOOP_POLICY=uvloop
|
||||
COPY ./lbry/scripts/wallet_server_entrypoint.sh /entrypoint.sh
|
||||
COPY ./docker/wallet_server_entrypoint.sh /entrypoint.sh
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -0,0 +1,33 @@
|
||||
version: "3"
|
||||
|
||||
volumes:
|
||||
lbrycrd:
|
||||
wallet_server:
|
||||
|
||||
services:
|
||||
lbrycrd:
|
||||
image: lbry/lbrycrd:${LBRYCRD_TAG:-latest-release}
|
||||
restart: always
|
||||
ports: # accessible from host
|
||||
- "9246:9246" # rpc port
|
||||
expose: # internal to docker network. also this doesn't do anything. its for documentation only.
|
||||
- "9245" # node-to-node comms port
|
||||
volumes:
|
||||
- "lbrycrd:/data/.lbrycrd"
|
||||
environment:
|
||||
- RUN_MODE=default
|
||||
- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
|
||||
- RPC_ALLOW_IP=0.0.0.0/0
|
||||
wallet_server:
|
||||
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
restart: always
|
||||
ports:
|
||||
- "50001:50001" # rpc port
|
||||
- "50005:50005" # websocket port
|
||||
volumes:
|
||||
- "wallet_server:/database"
|
||||
environment:
|
||||
- SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
|
||||
- DAEMON_URL=http://lbry:lbry@lbrycrd:9245
|
||||
@@ -0,0 +1,11 @@
|
||||
# requires powershell and .NET 4+. see https://chocolatey.org/install for more info.
|
||||
|
||||
$chocoVersion = powershell choco -v
|
||||
if(-not($chocoVersion)){
|
||||
Write-Output "Chocolatey is not installed, installing now"
|
||||
Write-Output "IF YOU KEEP GETTING THIS MESSAGE ON EVERY BUILD, TRY RESTARTING THE GITLAB RUNNER SO IT GETS CHOCO INTO IT'S ENV"
|
||||
Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
|
||||
}
|
||||
else{
|
||||
Write-Output "Chocolatey version $chocoVersion is already installed"
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Set the build version to be 'qa', 'rc', 'release'"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
|
||||
log = logging.getLogger()
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def get_build_type(ci_tag=None):
|
||||
if not ci_tag:
|
||||
return "qa"
|
||||
log.debug("getting build type for tag: \"%s\"", ci_tag)
|
||||
if re.match(r'v\d+\.\d+\.\d+rc\d+$', ci_tag):
|
||||
return 'rc'
|
||||
elif re.match(r'v\d+\.\d+\.\d+$', ci_tag):
|
||||
return 'release'
|
||||
return 'qa'
|
||||
|
||||
|
||||
def main():
|
||||
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
build_type_path = os.path.join(root_dir, 'lbry', 'lbry', 'build_type.py')
|
||||
log.debug("configuring build type file: %s", build_type_path)
|
||||
commit_hash = os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT'))
|
||||
if commit_hash is None:
|
||||
raise ValueError("Commit hash not found in env vars")
|
||||
commit_hash = commit_hash[:6]
|
||||
build_type = get_build_type(os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG')))
|
||||
log.debug("setting build type=%s, build commit=%s", build_type, commit_hash)
|
||||
with open(build_type_path, 'w') as f:
|
||||
f.write(f"BUILD = \"{build_type}\"\nBUILD_COMMIT = \"{commit_hash}\"\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
File renamed without changes.
@@ -11,3 +11,4 @@ __pycache__
|
||||
_trial_temp/
|
||||
|
||||
/tests/integration/files
|
||||
/tests/.coverage.*
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry-sdk) [](https://codecov.io/gh/lbryio/lbry-sdk)
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry-sdk)
|
||||
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
|
||||
+764
-144
File diff suppressed because it is too large.
Load diff
@@ -1,4 +1,4 @@
|
||||
__name__ = "lbry"
|
||||
__version__ = "0.46.1"
|
||||
__version__ = "0.48.2"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
@@ -163,7 +163,10 @@ class AbstractBlob:
|
||||
if not self.is_readable():
|
||||
raise OSError('blob files cannot be read')
|
||||
with self.reader_context() as handle:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
try:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError, AttributeError):
|
||||
return -1
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
|
||||
@@ -27,7 +27,7 @@ class BlobDownloader:
|
||||
self.ignored: typing.Dict['KademliaPeer', int] = {}
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
self.failures: typing.Dict['KademliaPeer', int] = {}
|
||||
self.connection_failures: typing.List['KademliaPeer'] = []
|
||||
self.connection_failures: typing.Set['KademliaPeer'] = set()
|
||||
self.connections: typing.Dict['KademliaPeer', 'BlobExchangeClientProtocol'] = {}
|
||||
self.is_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
@@ -48,7 +48,7 @@ class BlobDownloader:
|
||||
connection_id=connection_id, connection_manager=self.blob_manager.connection_manager
|
||||
)
|
||||
if not bytes_received and not protocol and peer not in self.connection_failures:
|
||||
self.connection_failures.append(peer)
|
||||
self.connection_failures.add(peer)
|
||||
if not protocol and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
@@ -87,32 +87,42 @@ class BlobDownloader:
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
self.is_running.set()
|
||||
tried_for_this_blob: typing.Set['KademliaPeer'] = set()
|
||||
try:
|
||||
while not blob.get_is_verified() and self.is_running.is_set():
|
||||
batch: typing.Set['KademliaPeer'] = set()
|
||||
batch: typing.Set['KademliaPeer'] = set(self.connections.keys())
|
||||
while not self.peer_queue.empty():
|
||||
batch.update(self.peer_queue.get_nowait())
|
||||
if batch:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
log.debug(
|
||||
"running, %d peers, %d ignored, %d active, %s connections",
|
||||
"%s running, %d peers, %d ignored, %d active, %s connections", blob_hash[:6],
|
||||
len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
|
||||
)
|
||||
re_add: typing.Set['KademliaPeer'] = set()
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if peer in self.ignored or peer in self.active_connections:
|
||||
if peer in self.ignored:
|
||||
continue
|
||||
if peer in tried_for_this_blob:
|
||||
continue
|
||||
if peer in self.active_connections:
|
||||
if peer not in re_add:
|
||||
re_add.add(peer)
|
||||
continue
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
just_probe = len(self.connections) == 0
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id, just_probe))
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
tried_for_this_blob.add(peer)
|
||||
if not re_add:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
finally:
|
||||
blob.close()
|
||||
if self.loop.is_running():
|
||||
self.loop.call_soon(self.cleanup_active)
|
||||
|
||||
def close(self):
|
||||
self.connection_failures.clear()
|
||||
|
||||
@@ -101,16 +101,21 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
self.started_transfer.set()
|
||||
try:
|
||||
sent = await asyncio.wait_for(blob.sendfile(self), self.transfer_timeout, loop=self.loop)
|
||||
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent)
|
||||
log.info("sent %s (%i bytes) to %s:%i", bh, sent, peer_address, peer_port)
|
||||
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError, asyncio.TimeoutError) as err:
|
||||
if sent and sent > 0:
|
||||
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent)
|
||||
log.info("sent %s (%i bytes) to %s:%i", bh, sent, peer_address, peer_port)
|
||||
else:
|
||||
log.debug("stopped sending %s to %s:%i", bh, peer_address, peer_port)
|
||||
except (OSError, asyncio.TimeoutError) as err:
|
||||
if isinstance(err, asyncio.TimeoutError):
|
||||
log.debug("timed out sending blob %s to %s", bh, peer_address)
|
||||
else:
|
||||
log.debug("stopped sending %s to %s:%i", bh, peer_address, peer_port)
|
||||
log.warning("could not read blob %s to send %s:%i", bh, peer_address, peer_port)
|
||||
self.close()
|
||||
finally:
|
||||
self.transfer_finished.set()
|
||||
else:
|
||||
log.info("don't have %s to send %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
if responses:
|
||||
self.send_response(responses)
|
||||
|
||||
|
||||
+6
-3
@@ -630,9 +630,13 @@ class Config(CLIConfig):
|
||||
|
||||
|
||||
def get_windows_directories() -> typing.Tuple[str, str, str]:
|
||||
from lbry.winpaths import get_path, FOLDERID, UserHandle # pylint: disable=import-outside-toplevel
|
||||
from lbry.winpaths import get_path, FOLDERID, UserHandle, \
|
||||
PathNotFoundException # pylint: disable=import-outside-toplevel
|
||||
|
||||
download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
try:
|
||||
download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
except PathNotFoundException:
|
||||
download_dir = os.getcwd()
|
||||
|
||||
# old
|
||||
appdata = get_path(FOLDERID.RoamingAppData, UserHandle.current)
|
||||
@@ -644,7 +648,6 @@ def get_windows_directories() -> typing.Tuple[str, str, str]:
|
||||
# new
|
||||
data_dir = user_data_dir('lbrynet', 'lbry')
|
||||
lbryum_dir = user_data_dir('lbryum', 'lbry')
|
||||
download_dir = get_path(FOLDERID.Downloads, UserHandle.current)
|
||||
return data_dir, lbryum_dir, download_dir
|
||||
|
||||
|
||||
|
||||
+69
-58
@@ -2,6 +2,7 @@ import logging
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
import socket
|
||||
from lbry.utils import resolve_host
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
@@ -19,7 +20,8 @@ log = logging.getLogger(__name__)
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.rpc_timeout,
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index):
|
||||
split_buckets_under_index: int = constants.split_buckets_under_index,
|
||||
storage: typing.Optional['SQLiteStorage'] = None):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
|
||||
@@ -28,6 +30,7 @@ class Node:
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
self._refresh_task: asyncio.Task = None
|
||||
self._storage = storage
|
||||
|
||||
async def refresh_node(self, force_once=False):
|
||||
while True:
|
||||
@@ -67,6 +70,8 @@ class Node:
|
||||
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
|
||||
if to_ping:
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
if self._storage:
|
||||
await self._storage.save_kademlia_peers(self.protocol.routing_table.get_peers())
|
||||
if force_once:
|
||||
break
|
||||
|
||||
@@ -111,7 +116,7 @@ class Node:
|
||||
self.listening_port = None
|
||||
log.info("Stopped DHT node")
|
||||
|
||||
async def start_listening(self, interface: str = '') -> None:
|
||||
async def start_listening(self, interface: str = '0.0.0.0') -> None:
|
||||
if not self.listening_port:
|
||||
self.listening_port, _ = await self.loop.create_datagram_endpoint(
|
||||
lambda: self.protocol, (interface, self.internal_udp_port)
|
||||
@@ -121,56 +126,53 @@ class Node:
|
||||
else:
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
async def join_network(self, interface: typing.Optional[str] = '',
|
||||
async def join_network(self, interface: str = '0.0.0.0',
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
def peers_from_urls(urls: typing.Optional[typing.List[typing.Tuple[bytes, str, int, int]]]):
|
||||
peer_addresses = []
|
||||
for node_id, address, udp_port, tcp_port in urls:
|
||||
if (node_id, address, udp_port, tcp_port) not in peer_addresses and \
|
||||
(address, udp_port) != (self.protocol.external_ip, self.protocol.udp_port):
|
||||
peer_addresses.append((node_id, address, udp_port, tcp_port))
|
||||
return [make_kademlia_peer(*peer_address) for peer_address in peer_addresses]
|
||||
|
||||
if not self.listening_port:
|
||||
await self.start_listening(interface)
|
||||
self.protocol.ping_queue.start()
|
||||
self._refresh_task = self.loop.create_task(self.refresh_node())
|
||||
|
||||
# resolve the known node urls
|
||||
known_node_addresses = []
|
||||
url_to_addr = {}
|
||||
|
||||
if known_node_urls:
|
||||
for host, port in known_node_urls:
|
||||
address = await resolve_host(host, port, proto='udp')
|
||||
if (address, port) not in known_node_addresses and\
|
||||
(address, port) != (self.protocol.external_ip, self.protocol.udp_port):
|
||||
known_node_addresses.append((address, port))
|
||||
url_to_addr[address] = host
|
||||
|
||||
if known_node_addresses:
|
||||
peers = [
|
||||
make_kademlia_peer(None, address, port)
|
||||
for (address, port) in known_node_addresses
|
||||
]
|
||||
while True:
|
||||
if not self.protocol.routing_table.get_peers():
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
self.protocol.peer_manager.reset()
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*peers, delay=0.0)
|
||||
peers.extend(await self.peer_search(self.protocol.node_id, shortlist=peers, count=32))
|
||||
if self.protocol.routing_table.get_peers():
|
||||
self.joined.set()
|
||||
log.info(
|
||||
"Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
else:
|
||||
while True:
|
||||
if self.protocol.routing_table.get_peers():
|
||||
if not self.joined.is_set():
|
||||
self.joined.set()
|
||||
log.info(
|
||||
"joined dht, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts()
|
||||
)
|
||||
else:
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
seed_peers = peers_from_urls(
|
||||
await self._storage.get_persisted_kademlia_peers()
|
||||
) if self._storage else []
|
||||
if not seed_peers:
|
||||
try:
|
||||
seed_peers.extend(peers_from_urls([
|
||||
(None, await resolve_host(address, udp_port, 'udp'), udp_port, None)
|
||||
for address, udp_port in known_node_urls or []
|
||||
]))
|
||||
except socket.gaierror:
|
||||
await asyncio.sleep(30, loop=self.loop)
|
||||
continue
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts())
|
||||
self.joined.set()
|
||||
self.protocol.peer_manager.reset()
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*seed_peers, delay=0.0)
|
||||
await self.peer_search(self.protocol.node_id, shortlist=seed_peers, count=32)
|
||||
|
||||
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
|
||||
self._join_task = self.loop.create_task(
|
||||
self.join_network(
|
||||
interface=interface, known_node_urls=known_node_urls
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
def start(self, interface: str, known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls))
|
||||
|
||||
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = constants.bottom_out_limit,
|
||||
@@ -197,25 +199,28 @@ class Node:
|
||||
peers.sort(key=lambda peer: distance(peer.node_id))
|
||||
return peers[:count]
|
||||
|
||||
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
|
||||
result_queue: asyncio.Queue):
|
||||
async def _accumulate_peers_for_value(self, search_queue: asyncio.Queue, result_queue: asyncio.Queue):
|
||||
tasks = []
|
||||
try:
|
||||
while True:
|
||||
blob_hash = await search_queue.get()
|
||||
tasks.append(self.loop.create_task(self._value_producer(blob_hash, result_queue)))
|
||||
tasks.append(self.loop.create_task(self._peers_for_value_producer(blob_hash, result_queue)))
|
||||
finally:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
|
||||
async def _value_producer(self, blob_hash: str, result_queue: asyncio.Queue):
|
||||
async def ping(_peer):
|
||||
async def _peers_for_value_producer(self, blob_hash: str, result_queue: asyncio.Queue):
|
||||
async def put_into_result_queue_after_pong(_peer):
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(_peer).ping()
|
||||
result_queue.put_nowait([_peer])
|
||||
log.debug("pong from %s:%i for %s", _peer.address, _peer.udp_port, blob_hash)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
# prioritize peers who reply to a dht ping first
|
||||
# this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers
|
||||
|
||||
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
|
||||
to_put = []
|
||||
for peer in results:
|
||||
@@ -223,19 +228,25 @@ class Node:
|
||||
continue
|
||||
is_good = self.protocol.peer_manager.peer_is_good(peer)
|
||||
if is_good:
|
||||
# the peer has replied recently over UDP, it can probably be reached on the TCP port
|
||||
to_put.append(peer)
|
||||
elif is_good is None:
|
||||
udp_port_to_try = 4444
|
||||
if 3400 > peer.tcp_port > 3332:
|
||||
if not peer.udp_port:
|
||||
udp_port_to_try = (peer.tcp_port - 3333) + 4444
|
||||
elif 4500 > peer.tcp_port > 4443:
|
||||
if not peer.udp_port:
|
||||
udp_port_to_try = peer.tcp_port
|
||||
if not peer.udp_port:
|
||||
peer = make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
|
||||
self.loop.create_task(ping(peer))
|
||||
# TODO: use the same port for TCP and UDP
|
||||
# the udp port must be guessed
|
||||
# default to the ports being the same. if the TCP port appears to be <=0.48.0 default,
|
||||
# including on a network with several nodes, then assume the udp port is proportionately
|
||||
# based on a starting port of 4444
|
||||
udp_port_to_try = peer.tcp_port
|
||||
if 3400 > peer.tcp_port > 3332:
|
||||
udp_port_to_try = (peer.tcp_port - 3333) + 4444
|
||||
self.loop.create_task(put_into_result_queue_after_pong(
|
||||
make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
|
||||
))
|
||||
else:
|
||||
self.loop.create_task(put_into_result_queue_after_pong(peer))
|
||||
else:
|
||||
# the peer is known to be bad/unreachable, skip trying to connect to it over TCP
|
||||
log.debug("skip bad peer %s:%i for %s", peer.address, peer.tcp_port, blob_hash)
|
||||
if to_put:
|
||||
result_queue.put_nowait(to_put)
|
||||
@@ -244,4 +255,4 @@ class Node:
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
|
||||
asyncio.Queue, asyncio.Task]:
|
||||
q = peer_queue or asyncio.Queue(loop=self.loop)
|
||||
return q, self.loop.create_task(self._accumulate_search_junction(search_queue, q))
|
||||
return q, self.loop.create_task(self._accumulate_peers_for_value(search_queue, q))
|
||||
+22
-6
@@ -15,14 +15,29 @@ log = logging.getLogger(__name__)
|
||||
@lru_cache(1024)
|
||||
def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str],
|
||||
udp_port: typing.Optional[int] = None,
|
||||
tcp_port: typing.Optional[int] = None) -> 'KademliaPeer':
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port)
|
||||
tcp_port: typing.Optional[int] = None,
|
||||
allow_localhost: bool = False) -> 'KademliaPeer':
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
|
||||
|
||||
|
||||
def is_valid_ipv4(address):
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
carrier_grade_NAT_subnet = ipaddress.ip_network('100.64.0.0/10')
|
||||
ip4_to_6_relay_subnet = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
|
||||
try:
|
||||
ip = ipaddress.ip_address(address)
|
||||
return ip.version == 4
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
|
||||
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
|
||||
parsed_ip.is_reserved,
|
||||
carrier_grade_NAT_subnet.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
ip4_to_6_relay_subnet.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
|
||||
@@ -151,6 +166,7 @@ class KademliaPeer:
|
||||
udp_port: typing.Optional[int] = field(hash=True)
|
||||
tcp_port: typing.Optional[int] = field(compare=False, hash=False)
|
||||
protocol_version: typing.Optional[int] = field(default=1, compare=False, hash=False)
|
||||
allow_localhost: bool = field(default=False, compare=False, hash=False)
|
||||
|
||||
def __post_init__(self):
|
||||
if self._node_id is not None:
|
||||
@@ -160,7 +176,7 @@ class KademliaPeer:
|
||||
raise ValueError("invalid udp port")
|
||||
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
|
||||
raise ValueError("invalid tcp port")
|
||||
if not is_valid_ipv4(self.address):
|
||||
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||
|
||||
def update_tcp_port(self, tcp_port: int):
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
from collections import defaultdict
|
||||
import typing
|
||||
import logging
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.dht.serialization.datagram import PAGE_KEY
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
if TYPE_CHECKING:
|
||||
@@ -46,6 +47,7 @@ class FindValueResponse(FindResponse):
|
||||
self.token = result_dict[b'token']
|
||||
self.close_triples: typing.List[typing.Tuple[bytes, bytes, int]] = result_dict.get(b'contacts', [])
|
||||
self.found_compact_addresses = result_dict.get(key, [])
|
||||
self.pages = int(result_dict.get(PAGE_KEY, 0))
|
||||
|
||||
@property
|
||||
def found(self) -> bool:
|
||||
@@ -151,7 +153,11 @@ class IterativeFinder:
|
||||
self._add_active(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
self._add_active(make_kademlia_peer(node_id, address, udp_port))
|
||||
try:
|
||||
self._add_active(make_kademlia_peer(node_id, address, udp_port))
|
||||
except ValueError:
|
||||
log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer.address,
|
||||
peer.udp_port, address, udp_port)
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
@@ -314,10 +320,41 @@ class IterativeValueFinder(IterativeFinder):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
shortlist)
|
||||
self.blob_peers: typing.Set['KademliaPeer'] = set()
|
||||
# this tracks the index of the most recent page we requested from each peer
|
||||
self.peer_pages: typing.DefaultDict['KademliaPeer', int] = defaultdict(int)
|
||||
# this tracks the set of blob peers returned by each peer
|
||||
self.discovered_peers: typing.Dict['KademliaPeer', typing.Set['KademliaPeer']] = defaultdict(set)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindValueResponse:
|
||||
response = await self.protocol.get_rpc_peer(peer).find_value(self.key)
|
||||
return FindValueResponse(self.key, response)
|
||||
page = self.peer_pages[peer]
|
||||
response = await self.protocol.get_rpc_peer(peer).find_value(self.key, page=page)
|
||||
parsed = FindValueResponse(self.key, response)
|
||||
if not parsed.found:
|
||||
return parsed
|
||||
already_known = len(self.discovered_peers[peer])
|
||||
decoded_peers = set()
|
||||
for compact_addr in parsed.found_compact_addresses:
|
||||
try:
|
||||
decoded_peers.add(self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr))
|
||||
except ValueError:
|
||||
log.warning("misbehaving peer %s:%i returned invalid peer for blob",
|
||||
peer.address, peer.udp_port)
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
parsed.found_compact_addresses.clear()
|
||||
return parsed
|
||||
self.discovered_peers[peer].update(decoded_peers)
|
||||
log.debug("probed %s:%i page %i, %i known", peer.address, peer.udp_port, page,
|
||||
already_known + len(parsed.found_compact_addresses))
|
||||
if len(self.discovered_peers[peer]) != already_known + len(parsed.found_compact_addresses):
|
||||
log.warning("misbehaving peer %s:%i returned duplicate peers for blob", peer.address, peer.udp_port)
|
||||
parsed.found_compact_addresses.clear()
|
||||
elif len(parsed.found_compact_addresses) >= constants.k and self.peer_pages[peer] < parsed.pages:
|
||||
# the peer returned a full page and indicates it has more
|
||||
self.peer_pages[peer] += 1
|
||||
if peer in self.contacted:
|
||||
# the peer must be removed from self.contacted so that it will be probed for the next page
|
||||
self.contacted.remove(peer)
|
||||
return parsed
|
||||
|
||||
def check_result_ready(self, response: FindValueResponse):
|
||||
if response.found:
|
||||
|
||||
@@ -1,202 +0,0 @@
|
||||
class RPCError(Exception):
|
||||
code = 0
|
||||
|
||||
|
||||
class PriceDisagreementError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DuplicateStreamHashError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DownloadCancelledError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DownloadSDTimeout(Exception):
|
||||
def __init__(self, download):
|
||||
super().__init__(f'Failed to download sd blob {download} within timeout')
|
||||
self.download = download
|
||||
|
||||
|
||||
class DownloadTimeoutError(Exception):
|
||||
def __init__(self, download):
|
||||
super().__init__(f'Failed to download {download} within timeout')
|
||||
self.download = download
|
||||
|
||||
|
||||
class DownloadDataTimeout(Exception):
|
||||
def __init__(self, download):
|
||||
super().__init__(f'Failed to download data blobs for sd hash {download} within timeout')
|
||||
self.download = download
|
||||
|
||||
|
||||
class ResolveTimeout(Exception):
|
||||
def __init__(self, uri):
|
||||
super().__init__(f'Failed to resolve "{uri}" within the timeout')
|
||||
self.uri = uri
|
||||
|
||||
|
||||
class RequestCanceledError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NegativeFundsError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NullFundsError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InsufficientFundsError(RPCError):
|
||||
code = -310
|
||||
|
||||
|
||||
class CurrencyConversionError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class FileOpenError(ValueError):
|
||||
# this extends ValueError because it is replacing a ValueError in EncryptedFileDownloader
|
||||
# and I don't know where it might get caught upstream
|
||||
pass
|
||||
|
||||
|
||||
class ResolveError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ConnectionClosedBeforeResponseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowed(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidExchangeRateResponse(Exception):
|
||||
def __init__(self, source, reason):
|
||||
super().__init__(f'Failed to get exchange rate from {source}:{reason}')
|
||||
self.source = source
|
||||
self.reason = reason
|
||||
|
||||
|
||||
class UnknownNameError(Exception):
|
||||
def __init__(self, name):
|
||||
super().__init__(f'Name {name} is unknown')
|
||||
self.name = name
|
||||
|
||||
|
||||
class UnknownClaimID(Exception):
|
||||
def __init__(self, claim_id):
|
||||
super().__init__(f'Claim {claim_id} is unknown')
|
||||
self.claim_id = claim_id
|
||||
|
||||
|
||||
class UnknownURI(Exception):
|
||||
def __init__(self, uri):
|
||||
super().__init__(f'URI {uri} cannot be resolved')
|
||||
self.name = uri
|
||||
|
||||
|
||||
class UnknownOutpoint(Exception):
|
||||
def __init__(self, outpoint):
|
||||
super().__init__(f'Outpoint {outpoint} cannot be resolved')
|
||||
self.outpoint = outpoint
|
||||
|
||||
|
||||
class InvalidName(Exception):
|
||||
def __init__(self, name, invalid_characters):
|
||||
self.name = name
|
||||
self.invalid_characters = invalid_characters
|
||||
super().__init__(
|
||||
'URI contains invalid characters: {}'.format(','.join(invalid_characters)))
|
||||
|
||||
|
||||
class UnknownStreamTypeError(Exception):
|
||||
def __init__(self, stream_type):
|
||||
self.stream_type = stream_type
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.stream_type)
|
||||
|
||||
|
||||
class InvalidStreamDescriptorError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidStreamInfoError(Exception):
|
||||
def __init__(self, name, stream_info):
|
||||
msg = f'{name} has claim with invalid stream info: {stream_info}'
|
||||
super().__init__(msg)
|
||||
self.name = name
|
||||
self.stream_info = stream_info
|
||||
|
||||
|
||||
class MisbehavingPeerError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidDataError(MisbehavingPeerError):
|
||||
pass
|
||||
|
||||
|
||||
class NoResponseError(MisbehavingPeerError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidResponseError(MisbehavingPeerError):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchBlobError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchStreamHash(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoSuchSDHash(Exception):
|
||||
"""
|
||||
Raised if sd hash is not known
|
||||
"""
|
||||
|
||||
|
||||
class InvalidBlobHashError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidHeaderError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidAuthenticationToken(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NegotiationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidCurrencyError(Exception):
|
||||
def __init__(self, currency):
|
||||
self.currency = currency
|
||||
super().__init__(
|
||||
f'Invalid currency: {currency} is not a supported currency.')
|
||||
|
||||
|
||||
class NoSuchDirectoryError(Exception):
|
||||
def __init__(self, directory):
|
||||
self.directory = directory
|
||||
super().__init__(f'No such directory {directory}')
|
||||
|
||||
|
||||
class ComponentStartConditionNotMet(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ComponentsNotStarted(Exception):
|
||||
pass
|
||||
@@ -0,0 +1,2 @@
|
||||
generate:
|
||||
python generate.py > __init__.py
|
||||
@@ -0,0 +1,86 @@
|
||||
Code | Name | Message | Comment
|
||||
---:|---|---|---
|
||||
**1xx** | Initialization | **Daemon `start` and other CLI command failures (non-recoverable)**
|
||||
**10x** | Client | Error codes reported by clients connecting to `lbrynet` daemon.
|
||||
101 | RPCConnection | Failed to establish HTTP connection to `lbrynet`. (Is it running?)
|
||||
102 | RPCUnresponsive | HTTP connection established but daemon is not responding to commands.
|
||||
103 | WebSocketConnection | WebSocket connection established but daemon is not responding to commands.
|
||||
104 | WebSocketUnresponsive | Failed to establish WebSocket connection to `lbrynet`. (Is it running?)
|
||||
**11x** | Hardware | Enough of `lbrynet` was able to start to determine external factors causing eventual failure.
|
||||
110 | OutOfSpace | Out of disk space.
|
||||
111 | OutOfRAM | Out of RAM.
|
||||
**12x** | Environment | Internal factors preventing `lbrynet` from bootstrapping itself.
|
||||
120 | IncompatiblePython | Incompatible version of Python.
|
||||
121 | IncompatibleDependency | Incompatible version of some library.
|
||||
**13x** | Configuration | Configuration errors.
|
||||
130 | CannotWriteConfiguration | Cannot write configuration file '{path}'. | When writing the default config fails on startup, such as due to permission issues.
|
||||
131 | CannotOpenConfiguration | Cannot find provided configuration file '{path}'. | Can't open the config file user provided via command line args.
|
||||
132 | CannotParseConfiguration | Failed to parse the configuration file '{path}'. | Includes the syntax error / line number to help user fix it.
|
||||
133 | ConfigurationMissing | Configuration file '{path}' is missing setting that has no default / fallback.
|
||||
134 | ConfigurationInvalid | Configuration file '{path}' has setting with invalid value.
|
||||
**14x** | Command | Errors preparing to execute commands.
|
||||
140 | CommandDoesNotExist | Command '{command}' does not exist.
|
||||
141 | CommandDeprecated | Command '{command}' is deprecated.
|
||||
142 | CommandInvalidArgument | Invalid arguments for command '{command}'.
|
||||
143 | CommandTemporarilyUnavailable | Command '{command}' is temporarily unavailable. | Such as waiting for required components to start.
|
||||
144 | CommandPermanentlyUnavailable | Command '{command}' is permanently unavailable. | such as when required component was intentionally configured not to start.
|
||||
**2xx** | Networking | **Networking**
|
||||
**20x** | Connectivity | General connectivity.
|
||||
201 | NoInternet | No internet connection.
|
||||
202 | NoUPnPSupport | Router does not support UPnP.
|
||||
**21x** | WalletConnectivity | Wallet server connectivity.
|
||||
210 | WalletConnection | Failed connecting to a lbryumx server. | Should normally not need to be handled higher up as `lbrynet` will retry other servers.
|
||||
211 | WalletConnections | Failed connecting to all known lbryumx servers. | Will need to bubble up and require user to do something.
|
||||
212 | WalletConnectionDropped | lbryumx droppped our connection. | Maybe we were being bad?
|
||||
**22x** | WalletDisconnected | Wallet connection dropped.
|
||||
220 | WalletServerSuspicious | Disconnected from lbryumx server due to suspicious responses. *generic*
|
||||
221 | WalletServerValidation | Disconnected from lbryumx server due to SPV validation failure.
|
||||
222 | WalletServerHeader | Disconnected from lbryumx server due to incorrect header received.
|
||||
228 | WalletServerVersion | Disconnected from lbryumx server due to incompatible protocol version.
|
||||
229 | WalletServerUnresponsive | Disconnected from lbryumx server due to unresponsiveness.
|
||||
**23x** | DataConnectivity | P2P connection errors.
|
||||
**24x** | DataNetwork | P2P download errors.
|
||||
240 | DataDownload | Failed to download blob. *generic*
|
||||
**25x** | DataUpload | P2P upload errors.
|
||||
**26x** | DHTConnectivity | DHT connectivity issues.
|
||||
**27x** | DHTProtocol | DHT protocol issues.
|
||||
**3xx** | Blockchain | **Blockchain**
|
||||
**30x** | TransactionRejection | Transaction rejected.
|
||||
300 | TransactionRejected | Transaction rejected, unknown reason.
|
||||
301 | TransactionFeeTooLow | Fee too low.
|
||||
302 | TransactionInvalidSignature | Invalid signature.
|
||||
**31x** | Balance | Errors related to your available balance.
|
||||
311 | InsufficientFunds | Insufficient funds. | determined by wallet prior to attempting to broadcast a tx; this is different for example from a TX being created and sent but then rejected by lbrycrd for unspendable utxos.
|
||||
**32x** | ChannelSigning | Channel signing.
|
||||
320 | ChannelKeyNotFound | Channel signing key not found.
|
||||
321 | ChannelKeyInvalid | Channel signing key is out of date. | For example, channel was updated but you don't have the updated key.
|
||||
**33x** | GeneralResolve | Errors while resolving urls.
|
||||
331 | Resolve | Failed to resolve '{url}'.
|
||||
332 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||
**4xx** | Blob | **Blobs**
|
||||
**40x** | BlobAvailability | Blob availability.
|
||||
400 | BlobNotFound | Blob not found.
|
||||
401 | BlobPermissionDenied | Permission denied to read blob.
|
||||
402 | BlobTooBig | Blob is too big.
|
||||
403 | BlobEmpty | Blob is empty.
|
||||
**41x** | BlobDecryption | Decryption / Assembly
|
||||
410 | BlobFailedDecryption | Failed to decrypt blob.
|
||||
411 | CorruptBlob | Blobs is corrupted.
|
||||
**42x** | BlobEncryption | Encrypting / Creating
|
||||
420 | BlobFailedEncryption | Failed to encrypt blob.
|
||||
**43x** | BlobRelated | Exceptions carried over from old error system.
|
||||
431 | DownloadCancelled | Download was canceled.
|
||||
432 | DownloadSDTimeout | Failed to download sd blob {download} within timeout.
|
||||
433 | DownloadDataTimeout | Failed to download data blobs for sd hash {download} within timeout.
|
||||
434 | InvalidStreamDescriptor | {message}
|
||||
435 | InvalidData | {message}
|
||||
436 | InvalidBlobHash | {message}
|
||||
**5xx** | Component | **Components**
|
||||
501 | ComponentStartConditionNotMet | Unresolved dependencies for: {components}
|
||||
502 | ComponentsNotStarted | {message}
|
||||
**6xx** | CurrencyExchange | **Currency Exchange**
|
||||
601 | InvalidExchangeRateResponse | Failed to get exchange rate from {source}: {reason}
|
||||
602 | CurrencyConversion | {message}
|
||||
603 | InvalidCurrency | Invalid currency: {currency} is not a supported currency.
|
||||
**7xx** | Purchase | Purchase process errors.
|
||||
701 | KeyFeeAboveMaxAllowed | {message}
|
||||
@@ -0,0 +1,481 @@
|
||||
from .base import BaseError
|
||||
|
||||
|
||||
class InitializationError(BaseError):
|
||||
"""
|
||||
**Daemon `start` and other CLI command failures (non-recoverable)**
|
||||
"""
|
||||
|
||||
|
||||
class ClientError(InitializationError):
|
||||
"""
|
||||
Error codes reported by clients connecting to `lbrynet` daemon.
|
||||
"""
|
||||
|
||||
|
||||
class RPCConnectionError(ClientError):
|
||||
def __init__(self):
|
||||
super().__init__("Failed to establish HTTP connection to `lbrynet`. (Is it running?)")
|
||||
|
||||
|
||||
class RPCUnresponsiveError(ClientError):
|
||||
def __init__(self):
|
||||
super().__init__("HTTP connection established but daemon is not responding to commands.")
|
||||
|
||||
|
||||
class WebSocketConnectionError(ClientError):
|
||||
def __init__(self):
|
||||
super().__init__("WebSocket connection established but daemon is not responding to commands.")
|
||||
|
||||
|
||||
class WebSocketUnresponsiveError(ClientError):
|
||||
def __init__(self):
|
||||
super().__init__("Failed to establish WebSocket connection to `lbrynet`. (Is it running?)")
|
||||
|
||||
|
||||
class HardwareError(InitializationError):
|
||||
"""
|
||||
Enough of `lbrynet` was able to start to determine external factors causing eventual failure.
|
||||
"""
|
||||
|
||||
|
||||
class OutOfSpaceError(HardwareError):
|
||||
def __init__(self):
|
||||
super().__init__("Out of disk space.")
|
||||
|
||||
|
||||
class OutOfRAMError(HardwareError):
|
||||
def __init__(self):
|
||||
super().__init__("Out of RAM.")
|
||||
|
||||
|
||||
class EnvironmentError(InitializationError):
|
||||
"""
|
||||
Internal factors preventing `lbrynet` from bootstrapping itself.
|
||||
"""
|
||||
|
||||
|
||||
class IncompatiblePythonError(EnvironmentError):
|
||||
def __init__(self):
|
||||
super().__init__("Incompatible version of Python.")
|
||||
|
||||
|
||||
class IncompatibleDependencyError(EnvironmentError):
|
||||
def __init__(self):
|
||||
super().__init__("Incompatible version of some library.")
|
||||
|
||||
|
||||
class ConfigurationError(InitializationError):
|
||||
"""
|
||||
Configuration errors.
|
||||
"""
|
||||
|
||||
|
||||
class CannotWriteConfigurationError(ConfigurationError):
|
||||
"""
|
||||
When writing the default config fails on startup, such as due to permission issues.
|
||||
"""
|
||||
def __init__(self, path):
|
||||
super().__init__(f"Cannot write configuration file '{path}'.")
|
||||
|
||||
|
||||
class CannotOpenConfigurationError(ConfigurationError):
|
||||
"""
|
||||
Can't open the config file user provided via command line args.
|
||||
"""
|
||||
def __init__(self, path):
|
||||
super().__init__(f"Cannot find provided configuration file '{path}'.")
|
||||
|
||||
|
||||
class CannotParseConfigurationError(ConfigurationError):
|
||||
"""
|
||||
Includes the syntax error / line number to help user fix it.
|
||||
"""
|
||||
def __init__(self, path):
|
||||
super().__init__(f"Failed to parse the configuration file '{path}'.")
|
||||
|
||||
|
||||
class ConfigurationMissingError(ConfigurationError):
|
||||
def __init__(self, path):
|
||||
super().__init__(f"Configuration file '{path}' is missing setting that has no default / fallback.")
|
||||
|
||||
|
||||
class ConfigurationInvalidError(ConfigurationError):
|
||||
def __init__(self, path):
|
||||
super().__init__(f"Configuration file '{path}' has setting with invalid value.")
|
||||
|
||||
|
||||
class CommandError(InitializationError):
|
||||
"""
|
||||
Errors preparing to execute commands.
|
||||
"""
|
||||
|
||||
|
||||
class CommandDoesNotExistError(CommandError):
|
||||
def __init__(self, command):
|
||||
super().__init__(f"Command '{command}' does not exist.")
|
||||
|
||||
|
||||
class CommandDeprecatedError(CommandError):
|
||||
def __init__(self, command):
|
||||
super().__init__(f"Command '{command}' is deprecated.")
|
||||
|
||||
|
||||
class CommandInvalidArgumentError(CommandError):
|
||||
def __init__(self, command):
|
||||
super().__init__(f"Invalid arguments for command '{command}'.")
|
||||
|
||||
|
||||
class CommandTemporarilyUnavailableError(CommandError):
|
||||
"""
|
||||
Such as waiting for required components to start.
|
||||
"""
|
||||
def __init__(self, command):
|
||||
super().__init__(f"Command '{command}' is temporarily unavailable.")
|
||||
|
||||
|
||||
class CommandPermanentlyUnavailableError(CommandError):
|
||||
"""
|
||||
such as when required component was intentionally configured not to start.
|
||||
"""
|
||||
def __init__(self, command):
|
||||
super().__init__(f"Command '{command}' is permanently unavailable.")
|
||||
|
||||
|
||||
class NetworkingError(BaseError):
|
||||
"""
|
||||
**Networking**
|
||||
"""
|
||||
|
||||
|
||||
class ConnectivityError(NetworkingError):
|
||||
"""
|
||||
General connectivity.
|
||||
"""
|
||||
|
||||
|
||||
class NoInternetError(ConnectivityError):
|
||||
def __init__(self):
|
||||
super().__init__("No internet connection.")
|
||||
|
||||
|
||||
class NoUPnPSupportError(ConnectivityError):
|
||||
def __init__(self):
|
||||
super().__init__("Router does not support UPnP.")
|
||||
|
||||
|
||||
class WalletConnectivityError(NetworkingError):
|
||||
"""
|
||||
Wallet server connectivity.
|
||||
"""
|
||||
|
||||
|
||||
class WalletConnectionError(WalletConnectivityError):
|
||||
"""
|
||||
Should normally not need to be handled higher up as `lbrynet` will retry other servers.
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__("Failed connecting to a lbryumx server.")
|
||||
|
||||
|
||||
class WalletConnectionsError(WalletConnectivityError):
|
||||
"""
|
||||
Will need to bubble up and require user to do something.
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__("Failed connecting to all known lbryumx servers.")
|
||||
|
||||
|
||||
class WalletConnectionDroppedError(WalletConnectivityError):
|
||||
"""
|
||||
Maybe we were being bad?
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__("lbryumx droppped our connection.")
|
||||
|
||||
|
||||
class WalletDisconnectedError(NetworkingError):
|
||||
"""
|
||||
Wallet connection dropped.
|
||||
"""
|
||||
|
||||
|
||||
class WalletServerSuspiciousError(WalletDisconnectedError):
|
||||
def __init__(self):
|
||||
super().__init__("Disconnected from lbryumx server due to suspicious responses. *generic*")
|
||||
|
||||
|
||||
class WalletServerValidationError(WalletDisconnectedError):
|
||||
def __init__(self):
|
||||
super().__init__("Disconnected from lbryumx server due to SPV validation failure.")
|
||||
|
||||
|
||||
class WalletServerHeaderError(WalletDisconnectedError):
|
||||
def __init__(self):
|
||||
super().__init__("Disconnected from lbryumx server due to incorrect header received.")
|
||||
|
||||
|
||||
class WalletServerVersionError(WalletDisconnectedError):
|
||||
def __init__(self):
|
||||
super().__init__("Disconnected from lbryumx server due to incompatible protocol version.")
|
||||
|
||||
|
||||
class WalletServerUnresponsiveError(WalletDisconnectedError):
|
||||
def __init__(self):
|
||||
super().__init__("Disconnected from lbryumx server due to unresponsiveness.")
|
||||
|
||||
|
||||
class DataConnectivityError(NetworkingError):
|
||||
"""
|
||||
P2P connection errors.
|
||||
"""
|
||||
|
||||
|
||||
class DataNetworkError(NetworkingError):
|
||||
"""
|
||||
P2P download errors.
|
||||
"""
|
||||
|
||||
|
||||
class DataDownloadError(DataNetworkError):
|
||||
def __init__(self):
|
||||
super().__init__("Failed to download blob. *generic*")
|
||||
|
||||
|
||||
class DataUploadError(NetworkingError):
|
||||
"""
|
||||
P2P upload errors.
|
||||
"""
|
||||
|
||||
|
||||
class DHTConnectivityError(NetworkingError):
|
||||
"""
|
||||
DHT connectivity issues.
|
||||
"""
|
||||
|
||||
|
||||
class DHTProtocolError(NetworkingError):
|
||||
"""
|
||||
DHT protocol issues.
|
||||
"""
|
||||
|
||||
|
||||
class BlockchainError(BaseError):
|
||||
"""
|
||||
**Blockchain**
|
||||
"""
|
||||
|
||||
|
||||
class TransactionRejectionError(BlockchainError):
|
||||
"""
|
||||
Transaction rejected.
|
||||
"""
|
||||
|
||||
|
||||
class TransactionRejectedError(TransactionRejectionError):
|
||||
def __init__(self):
|
||||
super().__init__("Transaction rejected, unknown reason.")
|
||||
|
||||
|
||||
class TransactionFeeTooLowError(TransactionRejectionError):
|
||||
def __init__(self):
|
||||
super().__init__("Fee too low.")
|
||||
|
||||
|
||||
class TransactionInvalidSignatureError(TransactionRejectionError):
|
||||
def __init__(self):
|
||||
super().__init__("Invalid signature.")
|
||||
|
||||
|
||||
class BalanceError(BlockchainError):
|
||||
"""
|
||||
Errors related to your available balance.
|
||||
"""
|
||||
|
||||
|
||||
class InsufficientFundsError(BalanceError):
|
||||
"""
|
||||
determined by wallet prior to attempting to broadcast a tx; this is different for example from a TX
|
||||
being created and sent but then rejected by lbrycrd for unspendable utxos.
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__("Insufficient funds.")
|
||||
|
||||
|
||||
class ChannelSigningError(BlockchainError):
|
||||
"""
|
||||
Channel signing.
|
||||
"""
|
||||
|
||||
|
||||
class ChannelKeyNotFoundError(ChannelSigningError):
|
||||
def __init__(self):
|
||||
super().__init__("Channel signing key not found.")
|
||||
|
||||
|
||||
class ChannelKeyInvalidError(ChannelSigningError):
|
||||
"""
|
||||
For example, channel was updated but you don't have the updated key.
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__("Channel signing key is out of date.")
|
||||
|
||||
|
||||
class GeneralResolveError(BlockchainError):
|
||||
"""
|
||||
Errors while resolving urls.
|
||||
"""
|
||||
|
||||
|
||||
class ResolveError(GeneralResolveError):
|
||||
def __init__(self, url):
|
||||
super().__init__(f"Failed to resolve '{url}'.")
|
||||
|
||||
|
||||
class ResolveTimeoutError(GeneralResolveError):
|
||||
def __init__(self, url):
|
||||
super().__init__(f"Failed to resolve '{url}' within the timeout.")
|
||||
|
||||
|
||||
class BlobError(BaseError):
|
||||
"""
|
||||
**Blobs**
|
||||
"""
|
||||
|
||||
|
||||
class BlobAvailabilityError(BlobError):
|
||||
"""
|
||||
Blob availability.
|
||||
"""
|
||||
|
||||
|
||||
class BlobNotFoundError(BlobAvailabilityError):
|
||||
def __init__(self):
|
||||
super().__init__("Blob not found.")
|
||||
|
||||
|
||||
class BlobPermissionDeniedError(BlobAvailabilityError):
|
||||
def __init__(self):
|
||||
super().__init__("Permission denied to read blob.")
|
||||
|
||||
|
||||
class BlobTooBigError(BlobAvailabilityError):
|
||||
def __init__(self):
|
||||
super().__init__("Blob is too big.")
|
||||
|
||||
|
||||
class BlobEmptyError(BlobAvailabilityError):
|
||||
def __init__(self):
|
||||
super().__init__("Blob is empty.")
|
||||
|
||||
|
||||
class BlobDecryptionError(BlobError):
|
||||
"""
|
||||
Decryption / Assembly
|
||||
"""
|
||||
|
||||
|
||||
class BlobFailedDecryptionError(BlobDecryptionError):
|
||||
def __init__(self):
|
||||
super().__init__("Failed to decrypt blob.")
|
||||
|
||||
|
||||
class CorruptBlobError(BlobDecryptionError):
|
||||
def __init__(self):
|
||||
super().__init__("Blobs is corrupted.")
|
||||
|
||||
|
||||
class BlobEncryptionError(BlobError):
|
||||
"""
|
||||
Encrypting / Creating
|
||||
"""
|
||||
|
||||
|
||||
class BlobFailedEncryptionError(BlobEncryptionError):
|
||||
def __init__(self):
|
||||
super().__init__("Failed to encrypt blob.")
|
||||
|
||||
|
||||
class BlobRelatedError(BlobError):
|
||||
"""
|
||||
Exceptions carried over from old error system.
|
||||
"""
|
||||
|
||||
|
||||
class DownloadCancelledError(BlobRelatedError):
|
||||
def __init__(self):
|
||||
super().__init__("Download was canceled.")
|
||||
|
||||
|
||||
class DownloadSDTimeoutError(BlobRelatedError):
|
||||
def __init__(self, download):
|
||||
super().__init__(f"Failed to download sd blob {download} within timeout.")
|
||||
|
||||
|
||||
class DownloadDataTimeoutError(BlobRelatedError):
|
||||
def __init__(self, download):
|
||||
super().__init__(f"Failed to download data blobs for sd hash {download} within timeout.")
|
||||
|
||||
|
||||
class InvalidStreamDescriptorError(BlobRelatedError):
|
||||
def __init__(self, message):
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidDataError(BlobRelatedError):
|
||||
def __init__(self, message):
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidBlobHashError(BlobRelatedError):
|
||||
def __init__(self, message):
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class ComponentError(BaseError):
|
||||
"""
|
||||
**Components**
|
||||
"""
|
||||
|
||||
|
||||
class ComponentStartConditionNotMetError(ComponentError):
|
||||
def __init__(self, components):
|
||||
super().__init__(f"Unresolved dependencies for: {components}")
|
||||
|
||||
|
||||
class ComponentsNotStartedError(ComponentError):
|
||||
def __init__(self, message):
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class CurrencyExchangeError(BaseError):
|
||||
"""
|
||||
**Currency Exchange**
|
||||
"""
|
||||
|
||||
|
||||
class InvalidExchangeRateResponseError(CurrencyExchangeError):
|
||||
def __init__(self, source, reason):
|
||||
super().__init__(f"Failed to get exchange rate from {source}: {reason}")
|
||||
|
||||
|
||||
class CurrencyConversionError(CurrencyExchangeError):
|
||||
def __init__(self, message):
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidCurrencyError(CurrencyExchangeError):
|
||||
def __init__(self, currency):
|
||||
super().__init__(f"Invalid currency: {currency} is not a supported currency.")
|
||||
|
||||
|
||||
class PurchaseError(BaseError):
|
||||
"""
|
||||
Purchase process errors.
|
||||
"""
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowedError(PurchaseError):
|
||||
def __init__(self, message):
|
||||
super().__init__(f"{message}")
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
class BaseError(Exception):
|
||||
pass
|
||||
@@ -0,0 +1,58 @@
|
||||
import re
|
||||
from textwrap import fill, indent
|
||||
|
||||
|
||||
CLASS = """
|
||||
class {name}Error({parent}Error):{doc}
|
||||
"""
|
||||
|
||||
INIT = """\
|
||||
def __init__({args}):
|
||||
super().__init__({format}"{desc}")
|
||||
"""
|
||||
|
||||
INDENT = ' ' * 4
|
||||
|
||||
|
||||
def main():
|
||||
with open('README.md', 'r') as readme:
|
||||
print('from .base import BaseError\n')
|
||||
stack = {}
|
||||
started = False
|
||||
for line in readme.readlines():
|
||||
if not started:
|
||||
started = line.startswith('---:|')
|
||||
continue
|
||||
if not line:
|
||||
break
|
||||
parent = 'Base'
|
||||
columns = [c.strip() for c in line.split('|')]
|
||||
(h, code, desc), comment = columns[:3], ""
|
||||
if len(columns) == 4:
|
||||
comment = columns[3].strip()
|
||||
if h.startswith('**'):
|
||||
if h.count('x') == 1:
|
||||
parent = stack[h[2:3]][0]
|
||||
stack[h.replace('**', '').replace('x', '')] = (code, desc)
|
||||
if h.count('x') == 2:
|
||||
stack[h.replace('**', '').replace('x', '')+'0'] = (code, desc)
|
||||
comment = f'\n{INDENT}"""\n{indent(fill(comment or desc, 100), INDENT)}\n{INDENT}"""'
|
||||
print(CLASS.format(name=code, parent=parent, doc=comment))
|
||||
continue
|
||||
parent = stack[h[:2]][0]
|
||||
args = ['self']
|
||||
for arg in re.findall('{([a-z0-1]+)}', desc):
|
||||
args.append(arg)
|
||||
fmt = ""
|
||||
if len(args) > 1:
|
||||
fmt = "f"
|
||||
if comment:
|
||||
comment = f'\n{INDENT}"""\n{indent(fill(comment, 100), INDENT)}\n{INDENT}"""'
|
||||
print((CLASS+INIT).format(
|
||||
name=code, parent=parent, args=', '.join(args),
|
||||
desc=desc, doc=comment, format=fmt
|
||||
))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,7 +1,7 @@
|
||||
import logging
|
||||
import asyncio
|
||||
from lbry.conf import Config
|
||||
from lbry.error import ComponentStartConditionNotMet
|
||||
from lbry.error import ComponentStartConditionNotMetError
|
||||
from lbry.dht.peer import PeerManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -106,7 +106,7 @@ class ComponentManager:
|
||||
staged.update(to_stage)
|
||||
steps.append(step)
|
||||
elif components:
|
||||
raise ComponentStartConditionNotMet("Unresolved dependencies for: %s" % components)
|
||||
raise ComponentStartConditionNotMetError(components)
|
||||
if reverse:
|
||||
steps.reverse()
|
||||
return steps
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import hashlib
|
||||
import math
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
import binascii
|
||||
import typing
|
||||
import base58
|
||||
@@ -13,6 +12,7 @@ from aioupnp.fault import UPnPError
|
||||
|
||||
from lbry import utils
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import is_valid_public_ipv4
|
||||
from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob_exchange.server import BlobServer
|
||||
@@ -21,7 +21,6 @@ from lbry.extras.daemon.Component import Component
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import LbryWalletManager
|
||||
from lbry.wallet.header import Headers
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,7 +28,6 @@ log = logging.getLogger(__name__)
|
||||
|
||||
DATABASE_COMPONENT = "database"
|
||||
BLOB_COMPONENT = "blob_manager"
|
||||
HEADERS_COMPONENT = "blockchain_headers"
|
||||
WALLET_COMPONENT = "wallet"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
@@ -52,7 +50,7 @@ class DatabaseComponent(Component):
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 12
|
||||
return 14
|
||||
|
||||
@property
|
||||
def revision_filename(self):
|
||||
@@ -97,135 +95,9 @@ class DatabaseComponent(Component):
|
||||
self.storage = None
|
||||
|
||||
|
||||
class HeadersComponent(Component):
|
||||
component_name = HEADERS_COMPONENT
|
||||
HEADERS_URL = "https://headers.lbry.io/blockchain_headers_latest"
|
||||
CHECKPOINT = ('100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2', 600_000)
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.headers_dir = os.path.join(self.conf.wallet_dir, 'lbc_mainnet')
|
||||
self.headers_file = os.path.join(self.headers_dir, 'headers')
|
||||
self.old_file = os.path.join(self.conf.wallet_dir, 'blockchain_headers')
|
||||
self.headers = Headers(self.headers_file)
|
||||
self.is_downloading_headers = False
|
||||
self._headers_progress_percent = 0
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
def _round_progress(self, local_height, remote_height):
|
||||
return min(max(math.ceil(float(local_height) / float(remote_height) * 100), 0), 100)
|
||||
|
||||
async def get_status(self) -> dict:
|
||||
progress = None
|
||||
if self.is_downloading_headers:
|
||||
progress = self._headers_progress_percent
|
||||
elif self.component_manager.has_component(WALLET_COMPONENT):
|
||||
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
if wallet_manager and wallet_manager.ledger.network.remote_height > 0:
|
||||
local_height = wallet_manager.ledger.headers.height
|
||||
remote_height = wallet_manager.ledger.network.remote_height
|
||||
progress = self._round_progress(local_height, remote_height)
|
||||
return {
|
||||
'downloading_headers': True,
|
||||
'download_progress': progress
|
||||
} if progress is not None and progress < 100 else {}
|
||||
|
||||
async def fetch_headers_from_s3(self):
|
||||
local_header_size = self.local_header_file_size()
|
||||
resume_header = {"Range": f"bytes={local_header_size}-"}
|
||||
async with utils.aiohttp_request('get', self.HEADERS_URL, headers=resume_header) as response:
|
||||
if response.status == 406 or response.content_length < self.headers.header_size: # our file is bigger
|
||||
log.warning("s3 is more out of date than we are")
|
||||
return
|
||||
final_size_after_download = response.content_length + local_header_size
|
||||
if final_size_after_download % self.headers.header_size != 0:
|
||||
log.warning("s3 appears to have corrupted header")
|
||||
return
|
||||
write_mode = "wb"
|
||||
if local_header_size > 0:
|
||||
log.info("Resuming download of %i bytes from s3", response.content_length)
|
||||
write_mode = "a+b"
|
||||
with open(self.headers_file, write_mode) as fd:
|
||||
while not response.content.at_eof():
|
||||
local_header_size += fd.write(await response.content.readany())
|
||||
self._headers_progress_percent = self._round_progress(
|
||||
local_header_size, final_size_after_download
|
||||
)
|
||||
|
||||
def local_header_file_size(self) -> int:
|
||||
if os.path.isfile(self.headers_file):
|
||||
return os.stat(self.headers_file).st_size
|
||||
return 0
|
||||
|
||||
async def get_downloadable_header_height(self) -> typing.Optional[int]:
|
||||
async with utils.aiohttp_request('HEAD', self.HEADERS_URL) as response:
|
||||
if response.status != 200:
|
||||
log.warning("Header download error, unexpected response code: %s", response.status)
|
||||
return -1
|
||||
return response.content_length // self.headers.header_size
|
||||
|
||||
async def should_download_headers_from_s3(self) -> bool:
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
return False
|
||||
s3_headers_depth = self.conf.s3_headers_depth
|
||||
if not s3_headers_depth:
|
||||
return False
|
||||
|
||||
local_height = self.local_header_file_size() // self.headers.header_size
|
||||
remote_height = await self.get_downloadable_header_height()
|
||||
if remote_height is not None:
|
||||
log.info("remote height: %i, local height: %i", remote_height, local_height)
|
||||
if remote_height > (local_height + s3_headers_depth):
|
||||
return True
|
||||
return False
|
||||
|
||||
def verify_checkpoint(self):
|
||||
expected_hash, at_height = self.CHECKPOINT
|
||||
if self.local_header_file_size() // self.headers.header_size < at_height:
|
||||
return False
|
||||
hash = hashlib.sha256()
|
||||
chunk_size = self.headers.header_size * 1000
|
||||
with open(self.headers_file, 'rb') as header_file:
|
||||
data = header_file.read(chunk_size)
|
||||
while data and header_file.tell() <= at_height * self.headers.header_size:
|
||||
hash.update(data)
|
||||
data = header_file.read(chunk_size)
|
||||
return hash.hexdigest() == expected_hash
|
||||
|
||||
async def start(self):
|
||||
if not os.path.exists(self.headers_dir):
|
||||
os.mkdir(self.headers_dir)
|
||||
if os.path.exists(self.old_file):
|
||||
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
|
||||
os.rename(self.old_file, self.headers_file)
|
||||
|
||||
try:
|
||||
if await self.should_download_headers_from_s3():
|
||||
self.is_downloading_headers = True
|
||||
await self.fetch_headers_from_s3()
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self.is_downloading_headers = False
|
||||
# fixme: workaround, this should happen before download but happens after because headers.connect fail
|
||||
if not self.verify_checkpoint():
|
||||
log.info("Checkpoint failed, verifying headers using slower method.")
|
||||
await self.headers.open()
|
||||
await self.headers.repair()
|
||||
await self.headers.close()
|
||||
else:
|
||||
log.info("Header checkpoint verified.")
|
||||
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class WalletComponent(Component):
|
||||
component_name = WALLET_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT, HEADERS_COMPONENT]
|
||||
depends_on = [DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
@@ -236,23 +108,36 @@ class WalletComponent(Component):
|
||||
return self.wallet_manager
|
||||
|
||||
async def get_status(self):
|
||||
if self.wallet_manager and self.wallet_manager.ledger.network.remote_height:
|
||||
local_height = self.wallet_manager.ledger.headers.height
|
||||
if self.wallet_manager is None:
|
||||
return
|
||||
session_pool = self.wallet_manager.ledger.network.session_pool
|
||||
sessions = session_pool.sessions
|
||||
result = {
|
||||
'servers': [
|
||||
{
|
||||
'host': session.server[0],
|
||||
'port': session.server[1],
|
||||
'latency': session.connection_latency,
|
||||
'availability': session.available,
|
||||
} for session in sessions
|
||||
],
|
||||
'known_servers': len(sessions),
|
||||
'available_servers': len(list(session_pool.available_sessions))
|
||||
}
|
||||
|
||||
if self.wallet_manager.ledger.network.remote_height:
|
||||
local_height = self.wallet_manager.ledger.local_height_including_downloaded_height
|
||||
remote_height = self.wallet_manager.ledger.network.remote_height
|
||||
best_hash = self.wallet_manager.get_best_blockhash()
|
||||
return {
|
||||
'connected_servers': [
|
||||
{
|
||||
'host': session.server[0],
|
||||
'port': session.server[1],
|
||||
'latency': round(session.connection_latency, 2),
|
||||
} for session in self.wallet_manager.ledger.network.session_pool.sessions
|
||||
if session and session.available
|
||||
],
|
||||
progress = min(max(math.ceil(float(local_height) / float(remote_height) * 100), 0), 100)
|
||||
result.update({
|
||||
'headers_synchronization_progress': progress,
|
||||
'blocks': max(local_height, 0),
|
||||
'blocks_behind': max(remote_height - local_height, 0),
|
||||
'best_blockhash': best_hash,
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
async def start(self):
|
||||
log.info("Starting torba wallet")
|
||||
@@ -304,7 +189,7 @@ class BlobComponent(Component):
|
||||
|
||||
class DHTComponent(Component):
|
||||
component_name = DHT_COMPONENT
|
||||
depends_on = [UPNP_COMPONENT]
|
||||
depends_on = [UPNP_COMPONENT, DATABASE_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
@@ -338,6 +223,7 @@ class DHTComponent(Component):
|
||||
self.external_peer_port = upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
|
||||
self.external_udp_port = upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
|
||||
external_ip = upnp_component.external_ip
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
if not external_ip:
|
||||
external_ip = await utils.get_external_ip()
|
||||
if not external_ip:
|
||||
@@ -352,11 +238,10 @@ class DHTComponent(Component):
|
||||
external_ip=external_ip,
|
||||
peer_port=self.external_peer_port,
|
||||
rpc_timeout=self.conf.node_rpc_timeout,
|
||||
split_buckets_under_index=self.conf.split_buckets_under_index
|
||||
)
|
||||
self.dht_node.start(
|
||||
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
|
||||
split_buckets_under_index=self.conf.split_buckets_under_index,
|
||||
storage=storage
|
||||
)
|
||||
self.dht_node.start(self.conf.network_interface, self.conf.known_dht_nodes)
|
||||
log.info("Started the dht")
|
||||
|
||||
async def stop(self):
|
||||
@@ -415,7 +300,7 @@ class StreamManagerComponent(Component):
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
node = self.component_manager.get_component(DHT_COMPONENT)\
|
||||
node = self.component_manager.get_component(DHT_COMPONENT) \
|
||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
@@ -501,9 +386,8 @@ class UPnPComponent(Component):
|
||||
log.info("got external ip from UPnP: %s", external_ip)
|
||||
except (asyncio.TimeoutError, UPnPError, NotImplementedError):
|
||||
pass
|
||||
|
||||
if external_ip == "0.0.0.0" or (external_ip and external_ip.startswith("192.")):
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.com fallback")
|
||||
if external_ip and not is_valid_public_ipv4(external_ip):
|
||||
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
|
||||
external_ip = await utils.get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
import logging
|
||||
import json
|
||||
@@ -20,12 +21,11 @@ from torba.client.wallet import Wallet, ENCRYPT_ON_DISK
|
||||
from torba.client.baseaccount import SingleKey, HierarchicalDeterministic
|
||||
|
||||
from lbry import utils
|
||||
from lbry.conf import Config, Setting
|
||||
from lbry.conf import Config, Setting, NOT_SET
|
||||
from lbry.blob.blob_file import is_valid_blobhash, BlobBuffer
|
||||
from lbry.blob_exchange.downloader import download_blob
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.error import DownloadSDTimeout, ComponentsNotStarted
|
||||
from lbry.error import NullFundsError, NegativeFundsError, ComponentStartConditionNotMet
|
||||
from lbry.error import DownloadSDTimeoutError, ComponentsNotStartedError, ComponentStartConditionNotMetError
|
||||
from lbry.extras import system_info
|
||||
from lbry.extras.daemon import analytics
|
||||
from lbry.extras.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
|
||||
@@ -67,10 +67,11 @@ def requires(*components, **conditions):
|
||||
for condition_name in condition_names:
|
||||
condition_result, err_msg = component_manager.evaluate_condition(condition_name)
|
||||
if not condition_result:
|
||||
raise ComponentStartConditionNotMet(err_msg)
|
||||
raise ComponentStartConditionNotMetError(err_msg)
|
||||
if not component_manager.all_components_running(*components):
|
||||
raise ComponentsNotStarted("the following required components have not yet started: "
|
||||
"%s" % json.dumps(components))
|
||||
raise ComponentsNotStartedError(
|
||||
f"the following required components have not yet started: {json.dumps(components)}"
|
||||
)
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
return _inner
|
||||
@@ -113,6 +114,8 @@ SHORT_ID_LEN = 20
|
||||
MAX_UPDATE_FEE_ESTIMATE = 0.3
|
||||
DEFAULT_PAGE_SIZE = 20
|
||||
|
||||
VALID_FULL_CLAIM_ID = re.compile('[0-9a-fA-F]{40}')
|
||||
|
||||
|
||||
def encode_pagination_doc(items):
|
||||
return {
|
||||
@@ -959,7 +962,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
save_file=save_file, wallet=wallet
|
||||
)
|
||||
if not stream:
|
||||
raise DownloadSDTimeout(uri)
|
||||
raise DownloadSDTimeoutError(uri)
|
||||
except Exception as e:
|
||||
log.warning("Error downloading %s: %s", uri, str(e))
|
||||
return {"error": str(e)}
|
||||
@@ -1004,6 +1007,23 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
setattr(c, key, cleaned)
|
||||
return {key: cleaned}
|
||||
|
||||
def jsonrpc_settings_clear(self, key):
|
||||
"""
|
||||
Clear daemon settings
|
||||
|
||||
Usage:
|
||||
settings_clear (<key>)
|
||||
|
||||
Options:
|
||||
None
|
||||
|
||||
Returns:
|
||||
(dict) Updated dictionary of daemon settings
|
||||
"""
|
||||
with self.conf.update_config() as c:
|
||||
setattr(c, key, NOT_SET)
|
||||
return {key: self.conf.settings_dict[key]}
|
||||
|
||||
PREFERENCE_DOC = """
|
||||
Preferences management.
|
||||
"""
|
||||
@@ -1289,10 +1309,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
accounts = wallet.get_accounts_or_all(funding_account_ids)
|
||||
|
||||
amount = self.get_dewies_or_error("amount", amount)
|
||||
if not amount:
|
||||
raise NullFundsError
|
||||
if amount < 0:
|
||||
raise NegativeFundsError()
|
||||
|
||||
if addresses and not isinstance(addresses, list):
|
||||
addresses = [addresses]
|
||||
@@ -2011,7 +2027,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--allow_duplicate_purchase] [--override_max_key_fee] [--preview] [--blocking]
|
||||
|
||||
Options:
|
||||
--claim_id=<claim_id> : (str) id of claim to purchase
|
||||
--claim_id=<claim_id> : (str) claim id of claim to purchase
|
||||
--url=<url> : (str) lookup claim to purchase by url
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
--funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction
|
||||
@@ -2058,15 +2074,17 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
def jsonrpc_claim_list(self, account_id=None, wallet_id=None, page=None, page_size=None):
|
||||
def jsonrpc_claim_list(self, claim_type=None, account_id=None, wallet_id=None, page=None, page_size=None):
|
||||
"""
|
||||
List my stream and channel claims.
|
||||
|
||||
Usage:
|
||||
claim_list [<account_id> | --account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
claim_list [--claim_type=<claim_type>]
|
||||
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--page=<page>] [--page_size=<page_size>]
|
||||
|
||||
Options:
|
||||
--claim_type=<claim_type> : (str) claim type: channel, stream, repost, collection
|
||||
--account_id=<account_id> : (str) id of the account to query
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
@@ -2082,7 +2100,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
else:
|
||||
claims = partial(self.ledger.get_claims, wallet=wallet, accounts=wallet.accounts)
|
||||
claim_count = partial(self.ledger.get_claim_count, wallet=wallet, accounts=wallet.accounts)
|
||||
return paginate_rows(claims, claim_count, page, page_size)
|
||||
return paginate_rows(claims, claim_count, page, page_size, claim_type=claim_type)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_claim_search(self, **kwargs):
|
||||
@@ -2094,7 +2112,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
eg. --height=">400000" would limit results to only claims above 400k block height.
|
||||
|
||||
Usage:
|
||||
claim_search [<name> | --name=<name>] [--txid=<txid>] [--nout=<nout>]
|
||||
claim_search [<name> | --name=<name>] [--text=<text>] [--txid=<txid>] [--nout=<nout>]
|
||||
[--claim_id=<claim_id> | --claim_ids=<claim_ids>...]
|
||||
[--channel=<channel> |
|
||||
[[--channel_ids=<channel_ids>...] [--not_channel_ids=<not_channel_ids>...]]]
|
||||
@@ -2107,6 +2125,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
[--support_amount=<support_amount>] [--trending_group=<trending_group>]
|
||||
[--trending_mixed=<trending_mixed>] [--trending_local=<trending_local>]
|
||||
[--trending_global=<trending_global]
|
||||
[--reposted_claim_id=<reposted_claim_id>] [--reposted=<reposted>]
|
||||
[--claim_type=<claim_type>] [--stream_types=<stream_types>...] [--media_types=<media_types>...]
|
||||
[--fee_currency=<fee_currency>] [--fee_amount=<fee_amount>]
|
||||
[--any_tags=<any_tags>...] [--all_tags=<all_tags>...] [--not_tags=<not_tags>...]
|
||||
@@ -2119,6 +2138,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Options:
|
||||
--name=<name> : (str) claim name (normalized)
|
||||
--text=<text> : (str) full text search
|
||||
--claim_id=<claim_id> : (str) full or partial claim id
|
||||
--claim_ids=<claim_ids> : (list) list of full claim ids
|
||||
--txid=<txid> : (str) transaction id
|
||||
@@ -2186,6 +2206,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
--trending_global=<trending_global>: (int) trending value calculated relative to all
|
||||
trending content globally (supports
|
||||
equality constraints)
|
||||
--reposted_claim_id=<reposted_claim_id>: (str) all reposts of the specified original claim id
|
||||
--reposted=<reposted> : (int) claims reposted this many times (supports
|
||||
equality constraints)
|
||||
--claim_type=<claim_type> : (str) filter by 'channel', 'stream' or 'unknown'
|
||||
--stream_types=<stream_types> : (list) filter by 'video', 'image', 'document', etc
|
||||
--media_types=<media_types> : (list) filter by 'video/mp4', 'image/png', etc
|
||||
@@ -2341,8 +2364,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
txo = tx.outputs[0]
|
||||
txo.generate_channel_private_key()
|
||||
|
||||
await tx.sign(funding_accounts)
|
||||
|
||||
if not preview:
|
||||
await tx.sign(funding_accounts)
|
||||
account.add_channel_private_key(txo.private_key)
|
||||
wallet.save()
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
@@ -2496,8 +2520,9 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
new_txo.script.generate()
|
||||
|
||||
await tx.sign(funding_accounts)
|
||||
|
||||
if not preview:
|
||||
await tx.sign(funding_accounts)
|
||||
account.add_channel_private_key(new_txo.private_key)
|
||||
wallet.save()
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
@@ -2811,6 +2836,78 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
f"to update a specific stream claim."
|
||||
)
|
||||
|
||||
@requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
|
||||
async def jsonrpc_stream_repost(self, name, bid, claim_id, allow_duplicate_name=False, channel_id=None,
|
||||
channel_name=None, channel_account_id=None, account_id=None, wallet_id=None,
|
||||
claim_address=None, funding_account_ids=None, preview=False, blocking=False):
|
||||
"""
|
||||
Creates a claim that references an existing stream by its claim id.
|
||||
|
||||
Usage:
|
||||
stream_repost (<name> | --name=<name>) (<bid> | --bid=<bid>) (<claim_id> | --claim_id=<claim_id>)
|
||||
[--allow_duplicate_name=<allow_duplicate_name>]
|
||||
[--channel_id=<channel_id> | --channel_name=<channel_name>]
|
||||
[--channel_account_id=<channel_account_id>...]
|
||||
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...]
|
||||
[--preview] [--blocking]
|
||||
|
||||
Options:
|
||||
--name=<name> : (str) name of the content (can only consist of a-z A-Z 0-9 and -(dash))
|
||||
--bid=<bid> : (decimal) amount to back the claim
|
||||
--claim_id=<claim_id> : (str) id of the claim being reposted
|
||||
--allow_duplicate_name=<allow_duplicate_name> : (bool) create new claim even if one already exists with
|
||||
given name. default: false.
|
||||
--channel_id=<channel_id> : (str) claim id of the publisher channel
|
||||
--channel_name=<channel_name> : (str) name of the publisher channel
|
||||
--channel_account_id=<channel_account_id>: (str) one or more account ids for accounts to look in
|
||||
for channel certificates, defaults to all accounts.
|
||||
--account_id=<account_id> : (str) account to use for holding the transaction
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
--funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction
|
||||
--claim_address=<claim_address>: (str) address where the claim is sent to, if not specified
|
||||
it will be determined automatically from the account
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until transaction is in mempool
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
self.valid_stream_name_or_error(name)
|
||||
account = wallet.get_account_or_default(account_id)
|
||||
funding_accounts = wallet.get_accounts_or_all(funding_account_ids)
|
||||
channel = await self.get_channel_or_none(wallet, channel_account_id, channel_id, channel_name, for_signing=True)
|
||||
amount = self.get_dewies_or_error('bid', bid, positive_value=True)
|
||||
claim_address = await self.get_receiving_address(claim_address, account)
|
||||
claims = await account.get_claims(claim_name=name)
|
||||
if len(claims) > 0:
|
||||
if not allow_duplicate_name:
|
||||
raise Exception(
|
||||
f"You already have a stream claim published under the name '{name}'. "
|
||||
f"Use --allow-duplicate-name flag to override."
|
||||
)
|
||||
if not VALID_FULL_CLAIM_ID.fullmatch(claim_id):
|
||||
raise Exception('Invalid claim id. It is expected to be a 40 characters long hexadecimal string.')
|
||||
|
||||
claim = Claim()
|
||||
claim.repost.reference.claim_id = claim_id
|
||||
tx = await Transaction.claim_create(
|
||||
name, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel
|
||||
)
|
||||
new_txo = tx.outputs[0]
|
||||
|
||||
if channel:
|
||||
new_txo.sign(channel)
|
||||
await tx.sign(funding_accounts)
|
||||
|
||||
if not preview:
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
await self.analytics_manager.send_claim_action('publish')
|
||||
else:
|
||||
await account.ledger.release_tx(tx)
|
||||
|
||||
return tx
|
||||
|
||||
@requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT)
|
||||
async def jsonrpc_stream_create(
|
||||
self, name, bid, file_path, allow_duplicate_name=False,
|
||||
@@ -3270,6 +3367,374 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
"""
|
||||
return self.get_est_cost_from_uri(uri)
|
||||
|
||||
COLLECTION_DOC = """
|
||||
Create, update, list, resolve, and abandon collections.
|
||||
"""
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_collection_create(
|
||||
self, name, bid, claims, allow_duplicate_name=False,
|
||||
channel_id=None, channel_name=None, channel_account_id=None,
|
||||
account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None,
|
||||
preview=False, blocking=False, **kwargs):
|
||||
"""
|
||||
Create a new collection.
|
||||
|
||||
Usage:
|
||||
collection_create (<name> | --name=<name>) (<bid> | --bid=<bid>)
|
||||
(<claims>... | --claims=<claims>...)
|
||||
[--allow_duplicate_name]
|
||||
[--title=<title>] [--description=<description>]
|
||||
[--tags=<tags>...] [--languages=<languages>...] [--locations=<locations>...]
|
||||
[--thumbnail_url=<thumbnail_url>]
|
||||
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--claim_address=<claim_address>] [--funding_account_ids=<funding_account_ids>...]
|
||||
[--preview] [--blocking]
|
||||
|
||||
Options:
|
||||
--name=<name> : (str) name of the collection
|
||||
--bid=<bid> : (decimal) amount to back the claim
|
||||
--claims=<claims> : (list) claim ids to be included in the collection
|
||||
--allow_duplicate_name : (bool) create new collection even if one already exists with
|
||||
given name. default: false.
|
||||
--title=<title> : (str) title of the collection
|
||||
--description=<description> : (str) description of the collection
|
||||
--clear_languages : (bool) clear existing languages (prior to adding new ones)
|
||||
--tags=<tags> : (list) content tags
|
||||
--clear_languages : (bool) clear existing languages (prior to adding new ones)
|
||||
--languages=<languages> : (list) languages used by the collection,
|
||||
using RFC 5646 format, eg:
|
||||
for English `--languages=en`
|
||||
for Spanish (Spain) `--languages=es-ES`
|
||||
for Spanish (Mexican) `--languages=es-MX`
|
||||
for Chinese (Simplified) `--languages=zh-Hans`
|
||||
for Chinese (Traditional) `--languages=zh-Hant`
|
||||
--locations=<locations> : (list) locations of the collection, consisting of 2 letter
|
||||
`country` code and a `state`, `city` and a postal
|
||||
`code` along with a `latitude` and `longitude`.
|
||||
for JSON RPC: pass a dictionary with aforementioned
|
||||
attributes as keys, eg:
|
||||
...
|
||||
"locations": [{'country': 'US', 'state': 'NH'}]
|
||||
...
|
||||
for command line: pass a colon delimited list
|
||||
with values in the following order:
|
||||
|
||||
"COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE"
|
||||
|
||||
making sure to include colon for blank values, for
|
||||
example to provide only the city:
|
||||
|
||||
... --locations="::Manchester"
|
||||
|
||||
with all values set:
|
||||
|
||||
... --locations="US:NH:Manchester:03101:42.990605:-71.460989"
|
||||
|
||||
optionally, you can just pass the "LATITUDE:LONGITUDE":
|
||||
|
||||
... --locations="42.990605:-71.460989"
|
||||
|
||||
finally, you can also pass JSON string of dictionary
|
||||
on the command line as you would via JSON RPC
|
||||
|
||||
... --locations="{'country': 'US', 'state': 'NH'}"
|
||||
|
||||
--thumbnail_url=<thumbnail_url>: (str) thumbnail url
|
||||
--account_id=<account_id> : (str) account to use for holding the transaction
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
--funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction
|
||||
--claim_address=<claim_address>: (str) address where the collection is sent to, if not specified
|
||||
it will be determined automatically from the account
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until transaction is in mempool
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
account = wallet.get_account_or_default(account_id)
|
||||
funding_accounts = wallet.get_accounts_or_all(funding_account_ids)
|
||||
self.valid_collection_name_or_error(name)
|
||||
channel = await self.get_channel_or_none(wallet, channel_account_id, channel_id, channel_name, for_signing=True)
|
||||
amount = self.get_dewies_or_error('bid', bid, positive_value=True)
|
||||
claim_address = await self.get_receiving_address(claim_address, account)
|
||||
|
||||
existing_collections = await self.ledger.get_collections(accounts=wallet.accounts, claim_name=name)
|
||||
if len(existing_collections) > 0:
|
||||
if not allow_duplicate_name:
|
||||
raise Exception(
|
||||
f"You already have a collection under the name '{name}'. "
|
||||
f"Use --allow-duplicate-name flag to override."
|
||||
)
|
||||
|
||||
claim = Claim()
|
||||
claim.collection.update(claims=claims, **kwargs)
|
||||
tx = await Transaction.claim_create(
|
||||
name, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel
|
||||
)
|
||||
new_txo = tx.outputs[0]
|
||||
|
||||
if channel:
|
||||
new_txo.sign(channel)
|
||||
await tx.sign(funding_accounts)
|
||||
|
||||
if not preview:
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
await self.analytics_manager.send_claim_action('publish')
|
||||
else:
|
||||
await account.ledger.release_tx(tx)
|
||||
|
||||
return tx
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_collection_update(
|
||||
self, claim_id, bid=None,
|
||||
channel_id=None, channel_name=None, channel_account_id=None, clear_channel=False,
|
||||
account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None,
|
||||
preview=False, blocking=False, replace=False, **kwargs):
|
||||
"""
|
||||
Update an existing collection claim.
|
||||
|
||||
Usage:
|
||||
collection_update (<claim_id> | --claim_id=<claim_id>) [--bid=<bid>]
|
||||
[--claims=<claims>...] [--clear_claims]
|
||||
[--title=<title>] [--description=<description>]
|
||||
[--tags=<tags>...] [--clear_tags]
|
||||
[--languages=<languages>...] [--clear_languages]
|
||||
[--locations=<locations>...] [--clear_locations]
|
||||
[--thumbnail_url=<thumbnail_url>] [--cover_url=<cover_url>]
|
||||
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--claim_address=<claim_address>] [--new_signing_key]
|
||||
[--funding_account_ids=<funding_account_ids>...]
|
||||
[--preview] [--blocking] [--replace]
|
||||
|
||||
Options:
|
||||
--claim_id=<claim_id> : (str) claim_id of the collection to update
|
||||
--bid=<bid> : (decimal) amount to back the claim
|
||||
--claims=<claims> : (list) claim ids
|
||||
--clear_claims : (bool) clear existing claim references (prior to adding new ones)
|
||||
--title=<title> : (str) title of the collection
|
||||
--description=<description> : (str) description of the collection
|
||||
--tags=<tags> : (list) add content tags
|
||||
--clear_tags : (bool) clear existing tags (prior to adding new ones)
|
||||
--languages=<languages> : (list) languages used by the collection,
|
||||
using RFC 5646 format, eg:
|
||||
for English `--languages=en`
|
||||
for Spanish (Spain) `--languages=es-ES`
|
||||
for Spanish (Mexican) `--languages=es-MX`
|
||||
for Chinese (Simplified) `--languages=zh-Hans`
|
||||
for Chinese (Traditional) `--languages=zh-Hant`
|
||||
--clear_languages : (bool) clear existing languages (prior to adding new ones)
|
||||
--locations=<locations> : (list) locations of the collection, consisting of 2 letter
|
||||
`country` code and a `state`, `city` and a postal
|
||||
`code` along with a `latitude` and `longitude`.
|
||||
for JSON RPC: pass a dictionary with aforementioned
|
||||
attributes as keys, eg:
|
||||
...
|
||||
"locations": [{'country': 'US', 'state': 'NH'}]
|
||||
...
|
||||
for command line: pass a colon delimited list
|
||||
with values in the following order:
|
||||
|
||||
"COUNTRY:STATE:CITY:CODE:LATITUDE:LONGITUDE"
|
||||
|
||||
making sure to include colon for blank values, for
|
||||
example to provide only the city:
|
||||
|
||||
... --locations="::Manchester"
|
||||
|
||||
with all values set:
|
||||
|
||||
... --locations="US:NH:Manchester:03101:42.990605:-71.460989"
|
||||
|
||||
optionally, you can just pass the "LATITUDE:LONGITUDE":
|
||||
|
||||
... --locations="42.990605:-71.460989"
|
||||
|
||||
finally, you can also pass JSON string of dictionary
|
||||
on the command line as you would via JSON RPC
|
||||
|
||||
... --locations="{'country': 'US', 'state': 'NH'}"
|
||||
|
||||
--clear_locations : (bool) clear existing locations (prior to adding new ones)
|
||||
--thumbnail_url=<thumbnail_url>: (str) thumbnail url
|
||||
--account_id=<account_id> : (str) account in which to look for collection (default: all)
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
--funding_account_ids=<funding_account_ids>: (list) ids of accounts to fund this transaction
|
||||
--claim_address=<claim_address>: (str) address where the collection is sent
|
||||
--new_signing_key : (bool) generate a new signing key, will invalidate all previous publishes
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until transaction is in mempool
|
||||
--replace : (bool) instead of modifying specific values on
|
||||
the collection, this will clear all existing values
|
||||
and only save passed in values, useful for form
|
||||
submissions where all values are always set
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
funding_accounts = wallet.get_accounts_or_all(funding_account_ids)
|
||||
if account_id:
|
||||
account = wallet.get_account_or_error(account_id)
|
||||
accounts = [account]
|
||||
else:
|
||||
account = wallet.default_account
|
||||
accounts = wallet.accounts
|
||||
|
||||
existing_collections = await self.ledger.get_collections(
|
||||
wallet=wallet, accounts=accounts, claim_id=claim_id
|
||||
)
|
||||
if len(existing_collections) != 1:
|
||||
account_ids = ', '.join(f"'{account.id}'" for account in accounts)
|
||||
raise Exception(
|
||||
f"Can't find the collection '{claim_id}' in account(s) {account_ids}."
|
||||
)
|
||||
old_txo = existing_collections[0]
|
||||
if not old_txo.claim.is_collection:
|
||||
raise Exception(
|
||||
f"A claim with id '{claim_id}' was found but it is not a collection."
|
||||
)
|
||||
|
||||
if bid is not None:
|
||||
amount = self.get_dewies_or_error('bid', bid, positive_value=True)
|
||||
else:
|
||||
amount = old_txo.amount
|
||||
|
||||
if claim_address is not None:
|
||||
self.valid_address_or_error(claim_address)
|
||||
else:
|
||||
claim_address = old_txo.get_address(account.ledger)
|
||||
|
||||
channel = None
|
||||
if channel_id or channel_name:
|
||||
channel = await self.get_channel_or_error(
|
||||
wallet, channel_account_id, channel_id, channel_name, for_signing=True)
|
||||
elif old_txo.claim.is_signed and not clear_channel and not replace:
|
||||
channel = old_txo.channel
|
||||
|
||||
if replace:
|
||||
claim = Claim()
|
||||
claim.collection.message.source.CopyFrom(
|
||||
old_txo.claim.collection.message.source
|
||||
)
|
||||
claim.collection.update(**kwargs)
|
||||
else:
|
||||
claim = Claim.from_bytes(old_txo.claim.to_bytes())
|
||||
claim.collection.update(**kwargs)
|
||||
tx = await Transaction.claim_update(
|
||||
old_txo, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel
|
||||
)
|
||||
new_txo = tx.outputs[0]
|
||||
|
||||
new_txo.script.generate()
|
||||
|
||||
if channel:
|
||||
new_txo.sign(channel)
|
||||
await tx.sign(funding_accounts)
|
||||
|
||||
if not preview:
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
await self.analytics_manager.send_claim_action('publish')
|
||||
else:
|
||||
await account.ledger.release_tx(tx)
|
||||
|
||||
return tx
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_collection_abandon(self, *args, **kwargs):
|
||||
"""
|
||||
Abandon one of my collection claims.
|
||||
|
||||
Usage:
|
||||
collection_abandon [<claim_id> | --claim_id=<claim_id>]
|
||||
[<txid> | --txid=<txid>] [<nout> | --nout=<nout>]
|
||||
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
|
||||
[--preview] [--blocking]
|
||||
|
||||
Options:
|
||||
--claim_id=<claim_id> : (str) claim_id of the claim to abandon
|
||||
--txid=<txid> : (str) txid of the claim to abandon
|
||||
--nout=<nout> : (int) nout of the claim to abandon
|
||||
--account_id=<account_id> : (str) id of the account to use
|
||||
--wallet_id=<wallet_id> : (str) restrict operation to specific wallet
|
||||
--preview : (bool) do not broadcast the transaction
|
||||
--blocking : (bool) wait until abandon is in mempool
|
||||
|
||||
Returns: {Transaction}
|
||||
"""
|
||||
return await self.jsonrpc_stream_abandon(*args, **kwargs)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
def jsonrpc_collection_list(self, resolve_claims=0, account_id=None, wallet_id=None, page=None, page_size=None):
|
||||
"""
|
||||
List my collection claims.
|
||||
|
||||
Usage:
|
||||
collection_list [--resolve_claims=<resolve_claims>] [<account_id> | --account_id=<account_id>]
|
||||
[--wallet_id=<wallet_id>] [--page=<page>] [--page_size=<page_size>]
|
||||
|
||||
Options:
|
||||
--resolve_claims=<resolve_claims> : (int) resolve every claim
|
||||
--account_id=<account_id> : (str) id of the account to use
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
--page_size=<page_size> : (int) number of items on page during pagination
|
||||
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
if account_id:
|
||||
account: LBCAccount = wallet.get_account_or_error(account_id)
|
||||
collections = account.get_collections
|
||||
collection_count = account.get_collection_count
|
||||
else:
|
||||
collections = partial(self.ledger.get_collections, wallet=wallet, accounts=wallet.accounts)
|
||||
collection_count = partial(self.ledger.get_collection_count, wallet=wallet, accounts=wallet.accounts)
|
||||
return paginate_rows(collections, collection_count, page, page_size, resolve_claims=resolve_claims)
|
||||
|
||||
async def jsonrpc_collection_resolve(
|
||||
self, claim_id=None, url=None, wallet_id=None, page=1, page_size=DEFAULT_PAGE_SIZE):
|
||||
"""
|
||||
Resolve claims in the collection.
|
||||
|
||||
Usage:
|
||||
collection_resolve (--claim_id=<claim_id> | --url=<url>)
|
||||
[--wallet_id=<wallet_id>] [--page=<page>] [--page_size=<page_size>]
|
||||
|
||||
Options:
|
||||
--claim_id=<claim_id> : (str) claim id of the collection
|
||||
--url=<url> : (str) url of the collection
|
||||
--wallet_id=<wallet_id> : (str) restrict results to specific wallet
|
||||
--page=<page> : (int) page to return during paginating
|
||||
--page_size=<page_size> : (int) number of items on page during pagination
|
||||
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(wallet_id)
|
||||
|
||||
if claim_id:
|
||||
txo = await self.ledger.get_claim_by_claim_id(wallet.accounts, claim_id)
|
||||
if not isinstance(txo, Output) or not txo.is_claim:
|
||||
raise Exception(f"Could not find collection with claim_id '{claim_id}'. ")
|
||||
elif url:
|
||||
txo = (await self.ledger.resolve(wallet.accounts, [url]))[url]
|
||||
if not isinstance(txo, Output) or not txo.is_claim:
|
||||
raise Exception(f"Could not find collection with url '{url}'. ")
|
||||
else:
|
||||
raise Exception(f"Missing argument claim_id or url. ")
|
||||
|
||||
page_num, page_size = abs(page), min(abs(page_size), 50)
|
||||
items = await self.ledger.resolve_collection(txo, page_size * (page_num - 1), page_size)
|
||||
total_items = len(txo.claim.collection.claims.ids)
|
||||
|
||||
return {
|
||||
"items": items,
|
||||
"total_pages": int((total_items + (page_size - 1)) / page_size),
|
||||
"total_items": total_items,
|
||||
"page_size": page_size,
|
||||
"page": page
|
||||
}
|
||||
|
||||
SUPPORT_DOC = """
|
||||
Create, list and abandon all types of supports.
|
||||
"""
|
||||
@@ -3313,7 +3778,6 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
)
|
||||
|
||||
if not preview:
|
||||
await tx.sign(funding_accounts)
|
||||
await self.broadcast_or_release(tx, blocking)
|
||||
await self.storage.save_supports({claim_id: [{
|
||||
'txid': tx.id,
|
||||
@@ -4152,6 +4616,21 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
except (TypeError, ValueError):
|
||||
raise Exception("Invalid stream name.")
|
||||
|
||||
@staticmethod
|
||||
def valid_collection_name_or_error(name: str):
|
||||
try:
|
||||
if not name:
|
||||
raise Exception('Collection name cannot be blank.')
|
||||
parsed = URL.parse(name)
|
||||
if parsed.has_channel:
|
||||
raise Exception(
|
||||
"Collection names cannot start with '@' symbol. This is reserved for channels claims."
|
||||
)
|
||||
if not parsed.has_stream or parsed.stream.name != name:
|
||||
raise Exception('Collection name has invalid characters.')
|
||||
except (TypeError, ValueError):
|
||||
raise Exception("Invalid collection name.")
|
||||
|
||||
@staticmethod
|
||||
def valid_channel_name_or_error(name: str):
|
||||
try:
|
||||
|
||||
@@ -5,7 +5,7 @@ import json
|
||||
from decimal import Decimal
|
||||
from typing import Optional
|
||||
from aiohttp.client_exceptions import ClientError
|
||||
from lbry.error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError
|
||||
from lbry.utils import aiohttp_request
|
||||
from lbry.wallet.dewies import lbc_to_dewies
|
||||
|
||||
@@ -81,7 +81,7 @@ class MarketFeed:
|
||||
try:
|
||||
response = await asyncio.wait_for(self._make_request(), self.REQUESTS_TIMEOUT)
|
||||
self._save_price(self._subtract_fee(self._handle_response(response)))
|
||||
except (asyncio.TimeoutError, InvalidExchangeRateResponse, ClientError) as err:
|
||||
except (asyncio.TimeoutError, InvalidExchangeRateResponseError, ClientError) as err:
|
||||
self._on_error(err)
|
||||
await asyncio.sleep(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
|
||||
|
||||
@@ -108,14 +108,14 @@ class BittrexFeed(MarketFeed):
|
||||
def _handle_response(self, response):
|
||||
json_response = json.loads(response)
|
||||
if 'result' not in json_response:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
trades = json_response['result']
|
||||
if len(trades) == 0:
|
||||
raise InvalidExchangeRateResponse(self.market, 'trades not found')
|
||||
raise InvalidExchangeRateResponseError(self.market, 'trades not found')
|
||||
totals = sum([i['Total'] for i in trades])
|
||||
qtys = sum([i['Quantity'] for i in trades])
|
||||
if totals <= 0 or qtys <= 0:
|
||||
raise InvalidExchangeRateResponse(self.market, 'quantities were not positive')
|
||||
raise InvalidExchangeRateResponseError(self.market, 'quantities were not positive')
|
||||
vwap = totals / qtys
|
||||
return float(1.0 / vwap)
|
||||
|
||||
@@ -133,7 +133,7 @@ class LBRYioFeed(MarketFeed):
|
||||
def _handle_response(self, response):
|
||||
json_response = json.loads(response)
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / json_response['data']['lbc_btc']
|
||||
|
||||
|
||||
@@ -151,9 +151,9 @@ class LBRYioBTCFeed(MarketFeed):
|
||||
try:
|
||||
json_response = json.loads(response)
|
||||
except ValueError:
|
||||
raise InvalidExchangeRateResponse(self.name, "invalid rate response : %s" % response)
|
||||
raise InvalidExchangeRateResponseError(self.name, "invalid rate response : %s" % response)
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / json_response['data']['btc_usd']
|
||||
|
||||
|
||||
@@ -171,10 +171,10 @@ class CryptonatorBTCFeed(MarketFeed):
|
||||
try:
|
||||
json_response = json.loads(response)
|
||||
except ValueError:
|
||||
raise InvalidExchangeRateResponse(self.name, "invalid rate response")
|
||||
raise InvalidExchangeRateResponseError(self.name, "invalid rate response")
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
@@ -192,10 +192,10 @@ class CryptonatorFeed(MarketFeed):
|
||||
try:
|
||||
json_response = json.loads(response)
|
||||
except ValueError:
|
||||
raise InvalidExchangeRateResponse(self.name, "invalid rate response")
|
||||
raise InvalidExchangeRateResponseError(self.name, "invalid rate response")
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
|
||||
@@ -28,14 +28,16 @@ def encode_txo_doc():
|
||||
'confirmations': "number of confirmed blocks",
|
||||
'is_change': "payment to change address, only available when it can be determined",
|
||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||
'type': "one of 'claim', 'support' or 'payment'",
|
||||
'type': "one of 'claim', 'support' or 'purchase'",
|
||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||
'claim_id': "when type is 'claim' or 'support', this is the claim id",
|
||||
'claim_id': "when type is 'claim', 'support' or 'purchase', this is the claim id",
|
||||
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
|
||||
'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload",
|
||||
'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc",
|
||||
'protobuf': "hex encoded raw protobuf version of 'value' field",
|
||||
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
|
||||
'claim': "for purchase outputs only, metadata of purchased claim",
|
||||
'reposted_claim': "for repost claims only, metadata of claim being reposted",
|
||||
'signing_channel': "for signed claims only, metadata of signing channel",
|
||||
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
|
||||
'purchase_receipt': "metadata for the purchase transaction associated with this claim"
|
||||
@@ -201,6 +203,10 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['short_url'] = output['meta'].pop('short_url')
|
||||
if 'canonical_url' in output['meta']:
|
||||
output['canonical_url'] = output['meta'].pop('canonical_url')
|
||||
if txo.claims is not None:
|
||||
output['claims'] = [self.encode_output(o) for o in txo.claims]
|
||||
if txo.reposted_claim is not None:
|
||||
output['reposted_claim'] = self.encode_output(txo.reposted_claim)
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
try:
|
||||
output['value'] = txo.claim
|
||||
|
||||
@@ -31,6 +31,10 @@ def migrate_db(conf, start, end):
|
||||
from .migrate10to11 import do_migration
|
||||
elif current == 11:
|
||||
from .migrate11to12 import do_migration
|
||||
elif current == 12:
|
||||
from .migrate12to13 import do_migration
|
||||
elif current == 13:
|
||||
from .migrate13to14 import do_migration
|
||||
else:
|
||||
raise Exception(f"DB migration of version {current} to {current+1} is not available")
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
current_columns = []
|
||||
for col_info in cursor.execute("pragma table_info('file');").fetchall():
|
||||
current_columns.append(col_info[1])
|
||||
if 'bt_infohash' in current_columns:
|
||||
connection.close()
|
||||
print("already migrated")
|
||||
return
|
||||
|
||||
cursor.executescript("""
|
||||
pragma foreign_keys=off;
|
||||
|
||||
create table if not exists torrent (
|
||||
bt_infohash char(20) not null primary key,
|
||||
tracker text,
|
||||
length integer not null,
|
||||
name text not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_node ( -- BEP-0005
|
||||
bt_infohash char(20) not null references torrent,
|
||||
host text not null,
|
||||
port integer not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_tracker ( -- BEP-0012
|
||||
bt_infohash char(20) not null references torrent,
|
||||
tracker text not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_http_seed ( -- BEP-0017
|
||||
bt_infohash char(20) not null references torrent,
|
||||
http_seed text not null
|
||||
);
|
||||
|
||||
create table if not exists new_file (
|
||||
stream_hash char(96) references stream,
|
||||
bt_infohash char(20) references torrent,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
status text not null,
|
||||
saved_file integer not null,
|
||||
content_fee text,
|
||||
added_on integer not null
|
||||
);
|
||||
|
||||
create table if not exists new_content_claim (
|
||||
stream_hash char(96) references stream,
|
||||
bt_infohash char(20) references torrent,
|
||||
claim_outpoint text unique not null references claim
|
||||
);
|
||||
|
||||
insert into new_file (stream_hash, bt_infohash, file_name, download_directory, blob_data_rate, status,
|
||||
saved_file, content_fee, added_on) select
|
||||
stream_hash, NULL, file_name, download_directory, blob_data_rate, status, saved_file, content_fee,
|
||||
added_on
|
||||
from file;
|
||||
|
||||
insert into new_content_claim (stream_hash, bt_infohash, claim_outpoint)
|
||||
select stream_hash, NULL, claim_outpoint from content_claim;
|
||||
|
||||
drop table file;
|
||||
drop table content_claim;
|
||||
alter table new_file rename to file;
|
||||
alter table new_content_claim rename to content_claim;
|
||||
|
||||
pragma foreign_keys=on;
|
||||
""")
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -0,0 +1,21 @@
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
cursor.executescript("""
|
||||
create table if not exists peer (
|
||||
node_id char(96) not null primary key,
|
||||
address text not null,
|
||||
udp_port integer not null,
|
||||
tcp_port integer,
|
||||
unique (address, udp_port)
|
||||
);
|
||||
""")
|
||||
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -28,12 +28,11 @@ def calculate_effective_amount(amount: str, supports: typing.Optional[typing.Lis
|
||||
)
|
||||
|
||||
|
||||
class StoredStreamClaim:
|
||||
def __init__(self, stream_hash: str, outpoint: opt_str = None, claim_id: opt_str = None, name: opt_str = None,
|
||||
class StoredContentClaim:
|
||||
def __init__(self, outpoint: opt_str = None, claim_id: opt_str = None, name: opt_str = None,
|
||||
amount: opt_int = None, height: opt_int = None, serialized: opt_str = None,
|
||||
channel_claim_id: opt_str = None, address: opt_str = None, claim_sequence: opt_int = None,
|
||||
channel_name: opt_str = None):
|
||||
self.stream_hash = stream_hash
|
||||
self.claim_id = claim_id
|
||||
self.outpoint = outpoint
|
||||
self.claim_name = name
|
||||
@@ -71,8 +70,16 @@ class StoredStreamClaim:
|
||||
}
|
||||
|
||||
|
||||
def _get_content_claims(transaction: sqlite3.Connection, query: str,
|
||||
source_hashes: typing.List[str]) -> typing.Dict[str, StoredContentClaim]:
|
||||
claims = {}
|
||||
for claim_info in _batched_select(transaction, query, source_hashes):
|
||||
claims[claim_info[0]] = StoredContentClaim(*claim_info[1:])
|
||||
return claims
|
||||
|
||||
|
||||
def get_claims_from_stream_hashes(transaction: sqlite3.Connection,
|
||||
stream_hashes: typing.List[str]) -> typing.Dict[str, StoredStreamClaim]:
|
||||
stream_hashes: typing.List[str]) -> typing.Dict[str, StoredContentClaim]:
|
||||
query = (
|
||||
"select content_claim.stream_hash, c.*, case when c.channel_claim_id is not null then "
|
||||
" (select claim_name from claim where claim_id==c.channel_claim_id) "
|
||||
@@ -81,13 +88,20 @@ def get_claims_from_stream_hashes(transaction: sqlite3.Connection,
|
||||
" inner join claim c on c.claim_outpoint=content_claim.claim_outpoint and content_claim.stream_hash in {}"
|
||||
" order by c.rowid desc"
|
||||
)
|
||||
return {
|
||||
claim_info.stream_hash: claim_info
|
||||
for claim_info in [
|
||||
None if not claim_info else StoredStreamClaim(*claim_info)
|
||||
for claim_info in _batched_select(transaction, query, stream_hashes)
|
||||
]
|
||||
}
|
||||
return _get_content_claims(transaction, query, stream_hashes)
|
||||
|
||||
|
||||
def get_claims_from_torrent_info_hashes(transaction: sqlite3.Connection,
|
||||
info_hashes: typing.List[str]) -> typing.Dict[str, StoredContentClaim]:
|
||||
query = (
|
||||
"select content_claim.bt_infohash, c.*, case when c.channel_claim_id is not null then "
|
||||
" (select claim_name from claim where claim_id==c.channel_claim_id) "
|
||||
" else null end as channel_name "
|
||||
" from content_claim "
|
||||
" inner join claim c on c.claim_outpoint=content_claim.claim_outpoint and content_claim.bt_infohash in {}"
|
||||
" order by c.rowid desc"
|
||||
)
|
||||
return _get_content_claims(transaction, query, info_hashes)
|
||||
|
||||
|
||||
def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
@@ -97,44 +111,55 @@ def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
yield from transaction.execute(query.format(bind), current_batch)
|
||||
|
||||
|
||||
def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
|
||||
raw_content_fee):
|
||||
return {
|
||||
"rowid": rowid,
|
||||
"added_on": added_on,
|
||||
"stream_hash": stream_hash,
|
||||
"file_name": file_name, # hex
|
||||
"download_directory": download_dir, # hex
|
||||
"blob_data_rate": data_rate,
|
||||
"status": status,
|
||||
"sd_hash": sd_hash,
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name, # hex
|
||||
"suggested_file_name": suggested_file_name, # hex
|
||||
"claim": claim,
|
||||
"saved_file": bool(saved_file),
|
||||
"content_fee": None if not raw_content_fee else Transaction(
|
||||
binascii.unhexlify(raw_content_fee)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Dict]:
|
||||
files = []
|
||||
signed_claims = {}
|
||||
stream_hashes_and_bt_infohashes = transaction.execute("select stream_hash, bt_infohash from file").fetchall()
|
||||
stream_hashes = tuple(
|
||||
stream_hash for (stream_hash,) in transaction.execute("select stream_hash from file").fetchall()
|
||||
stream_hash for stream_hash, _ in stream_hashes_and_bt_infohashes if stream_hash is not None
|
||||
)
|
||||
for (rowid, stream_hash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee, added_on,
|
||||
_, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
for (rowid, stream_hash, bt_infohash, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
|
||||
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in _batched_select(
|
||||
transaction, "select file.rowid, file.*, stream.*, c.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"where file.stream_hash in {} "
|
||||
"order by c.rowid desc", stream_hashes):
|
||||
claim = StoredStreamClaim(stream_hash, *claim_args)
|
||||
claim = StoredContentClaim(*claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
signed_claims[claim.channel_claim_id] = []
|
||||
signed_claims[claim.channel_claim_id].append(claim)
|
||||
files.append(
|
||||
{
|
||||
"rowid": rowid,
|
||||
"added_on": added_on,
|
||||
"stream_hash": stream_hash,
|
||||
"file_name": file_name, # hex
|
||||
"download_directory": download_dir, # hex
|
||||
"blob_data_rate": data_rate,
|
||||
"status": status,
|
||||
"sd_hash": sd_hash,
|
||||
"key": stream_key,
|
||||
"stream_name": stream_name, # hex
|
||||
"suggested_file_name": suggested_file_name, # hex
|
||||
"claim": claim,
|
||||
"saved_file": bool(saved_file),
|
||||
"content_fee": None if not raw_content_fee else Transaction(
|
||||
binascii.unhexlify(raw_content_fee)
|
||||
)
|
||||
}
|
||||
_get_lbry_file_stream_dict(
|
||||
rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
|
||||
raw_content_fee
|
||||
)
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
transaction, "select c.claim_name, c.claim_id from claim c where c.claim_id in {}",
|
||||
@@ -179,6 +204,15 @@ def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor
|
||||
transaction.executemany("delete from blob where blob_hash=?", blob_hashes).fetchall()
|
||||
|
||||
|
||||
def delete_torrent(transaction: sqlite3.Connection, bt_infohash: str):
|
||||
transaction.execute("delete from content_claim where bt_infohash=?", (bt_infohash, )).fetchall()
|
||||
transaction.execute("delete from torrent_tracker where bt_infohash=?", (bt_infohash,)).fetchall()
|
||||
transaction.execute("delete from torrent_node where bt_infohash=?", (bt_infohash,)).fetchall()
|
||||
transaction.execute("delete from torrent_http_seed where bt_infohash=?", (bt_infohash,)).fetchall()
|
||||
transaction.execute("delete from file where bt_infohash=?", (bt_infohash,)).fetchall()
|
||||
transaction.execute("delete from torrent where bt_infohash=?", (bt_infohash,)).fetchall()
|
||||
|
||||
|
||||
def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typing.Optional[str],
|
||||
download_directory: typing.Optional[str], data_payment_rate: float, status: str,
|
||||
content_fee: typing.Optional[Transaction], added_on: typing.Optional[int] = None) -> int:
|
||||
@@ -189,12 +223,10 @@ def store_file(transaction: sqlite3.Connection, stream_hash: str, file_name: typ
|
||||
encoded_download_dir = binascii.hexlify(download_directory.encode()).decode()
|
||||
time_added = added_on or int(time.time())
|
||||
transaction.execute(
|
||||
"insert or replace into file values (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
"insert or replace into file values (?, NULL, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(stream_hash, encoded_file_name, encoded_download_dir, data_payment_rate, status,
|
||||
1 if (file_name and download_directory and os.path.isfile(os.path.join(download_directory, file_name))) else 0,
|
||||
None if not content_fee else binascii.hexlify(content_fee.raw).decode(),
|
||||
time_added
|
||||
)
|
||||
None if not content_fee else binascii.hexlify(content_fee.raw).decode(), time_added)
|
||||
).fetchall()
|
||||
|
||||
return transaction.execute("select rowid from file where stream_hash=?", (stream_hash, )).fetchone()[0]
|
||||
@@ -243,8 +275,32 @@ class SQLiteStorage(SQLiteMixin):
|
||||
claim_sequence integer not null
|
||||
);
|
||||
|
||||
create table if not exists torrent (
|
||||
bt_infohash char(20) not null primary key,
|
||||
tracker text,
|
||||
length integer not null,
|
||||
name text not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_node ( -- BEP-0005
|
||||
bt_infohash char(20) not null references torrent,
|
||||
host text not null,
|
||||
port integer not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_tracker ( -- BEP-0012
|
||||
bt_infohash char(20) not null references torrent,
|
||||
tracker text not null
|
||||
);
|
||||
|
||||
create table if not exists torrent_http_seed ( -- BEP-0017
|
||||
bt_infohash char(20) not null references torrent,
|
||||
http_seed text not null
|
||||
);
|
||||
|
||||
create table if not exists file (
|
||||
stream_hash text primary key not null references stream,
|
||||
stream_hash char(96) references stream,
|
||||
bt_infohash char(20) references torrent,
|
||||
file_name text,
|
||||
download_directory text,
|
||||
blob_data_rate real not null,
|
||||
@@ -255,9 +311,9 @@ class SQLiteStorage(SQLiteMixin):
|
||||
);
|
||||
|
||||
create table if not exists content_claim (
|
||||
stream_hash text unique not null references file,
|
||||
claim_outpoint text not null references claim,
|
||||
primary key (stream_hash, claim_outpoint)
|
||||
stream_hash char(96) references stream,
|
||||
bt_infohash char(20) references torrent,
|
||||
claim_outpoint text unique not null references claim
|
||||
);
|
||||
|
||||
create table if not exists support (
|
||||
@@ -273,6 +329,14 @@ class SQLiteStorage(SQLiteMixin):
|
||||
timestamp integer,
|
||||
primary key (sd_hash, reflector_address)
|
||||
);
|
||||
|
||||
create table if not exists peer (
|
||||
node_id char(96) not null primary key,
|
||||
address text not null,
|
||||
udp_port integer not null,
|
||||
tcp_port integer,
|
||||
unique (address, udp_port)
|
||||
);
|
||||
"""
|
||||
|
||||
def __init__(self, conf: Config, path, loop=None, time_getter: typing.Optional[typing.Callable[[], float]] = None):
|
||||
@@ -449,6 +513,9 @@ class SQLiteStorage(SQLiteMixin):
|
||||
def delete_stream(self, descriptor: 'StreamDescriptor'):
|
||||
return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
|
||||
|
||||
async def delete_torrent(self, bt_infohash: str):
|
||||
return await self.db.run(delete_torrent, bt_infohash)
|
||||
|
||||
# # # # # # # # # file stuff # # # # # # # # #
|
||||
|
||||
def save_downloaded_file(self, stream_hash: str, file_name: typing.Optional[str],
|
||||
@@ -475,7 +542,8 @@ class SQLiteStorage(SQLiteMixin):
|
||||
def update_manually_removed_files(transaction: sqlite3.Connection):
|
||||
removed = []
|
||||
for (stream_hash, download_directory, file_name) in transaction.execute(
|
||||
"select stream_hash, download_directory, file_name from file where saved_file=1"
|
||||
"select stream_hash, download_directory, file_name from file where saved_file=1 "
|
||||
"and stream_hash is not null"
|
||||
).fetchall():
|
||||
if download_directory and file_name and os.path.isfile(
|
||||
os.path.join(binascii.unhexlify(download_directory).decode(),
|
||||
@@ -536,7 +604,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
store_file(transaction, descriptor.stream_hash, os.path.basename(descriptor.suggested_file_name),
|
||||
download_directory, 0.0, 'stopped', content_fee=content_fee)
|
||||
if content_claim:
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?)", content_claim)
|
||||
transaction.execute("insert or ignore into content_claim values (?, ?, ?)", content_claim)
|
||||
transaction.executemany(
|
||||
"update file set status='stopped' where stream_hash=?",
|
||||
((stream_hash, ) for stream_hash in stream_hashes)
|
||||
@@ -703,8 +771,9 @@ class SQLiteStorage(SQLiteMixin):
|
||||
)
|
||||
|
||||
# update the claim associated to the file
|
||||
transaction.execute("delete from content_claim where stream_hash=?", (stream_hash, )).fetchall()
|
||||
transaction.execute(
|
||||
"insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint)
|
||||
"insert into content_claim values (?, NULL, ?)", (stream_hash, claim_outpoint)
|
||||
).fetchall()
|
||||
|
||||
async def save_content_claim(self, stream_hash, claim_outpoint):
|
||||
@@ -744,3 +813,17 @@ class SQLiteStorage(SQLiteMixin):
|
||||
"where r.timestamp is null or r.timestamp < ?",
|
||||
int(self.time_getter()) - 86400
|
||||
)
|
||||
|
||||
# # # # # # # # # # dht functions # # # # # # # # # # #
|
||||
async def get_persisted_kademlia_peers(self) -> typing.List[typing.Tuple[bytes, str, int, int]]:
|
||||
query = 'select node_id, address, udp_port, tcp_port from peer'
|
||||
return [(binascii.unhexlify(n), a, u, t) for n, a, u, t in await self.db.execute_fetchall(query)]
|
||||
|
||||
async def save_kademlia_peers(self, peers: typing.List['KademliaPeer']):
|
||||
def _save_kademlia_peers(transaction: sqlite3.Connection):
|
||||
transaction.execute('delete from peer').fetchall()
|
||||
transaction.executemany(
|
||||
'insert into peer(node_id, address, udp_port, tcp_port) values (?, ?, ?, ?)',
|
||||
tuple([(binascii.hexlify(p.node_id), p.address, p.udp_port, p.tcp_port) for p in peers])
|
||||
).fetchall()
|
||||
return await self.db.run(_save_kademlia_peers)
|
||||
@@ -191,6 +191,22 @@ class Source(Metadata):
|
||||
def sd_hash_bytes(self, sd_hash: bytes):
|
||||
self.message.sd_hash = sd_hash
|
||||
|
||||
@property
|
||||
def bt_infohash(self) -> str:
|
||||
return hexlify(self.message.bt_infohash).decode()
|
||||
|
||||
@bt_infohash.setter
|
||||
def bt_infohash(self, bt_infohash: str):
|
||||
self.message.bt_infohash = unhexlify(bt_infohash.encode())
|
||||
|
||||
@property
|
||||
def bt_infohash_bytes(self) -> bytes:
|
||||
return self.message.bt_infohash.decode()
|
||||
|
||||
@bt_infohash_bytes.setter
|
||||
def bt_infohash_bytes(self, bt_infohash: bytes):
|
||||
self.message.bt_infohash = bt_infohash
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return self.message.url
|
||||
|
||||
@@ -195,6 +195,8 @@ class Stream(BaseClaim):
|
||||
claim['source']['hash'] = self.source.file_hash
|
||||
if 'sd_hash' in claim['source']:
|
||||
claim['source']['sd_hash'] = self.source.sd_hash
|
||||
elif 'bt_infohash' in claim['source']:
|
||||
claim['source']['bt_infohash'] = self.source.bt_infohash
|
||||
if 'media_type' in claim['source']:
|
||||
claim['stream_type'] = guess_stream_type(claim['source']['media_type'])
|
||||
fee = claim.get('fee', {})
|
||||
@@ -216,6 +218,8 @@ class Stream(BaseClaim):
|
||||
|
||||
if 'sd_hash' in kwargs:
|
||||
self.source.sd_hash = kwargs.pop('sd_hash')
|
||||
elif 'bt_infohash' in kwargs:
|
||||
self.source.bt_infohash = kwargs.pop('bt_infohash')
|
||||
if 'file_name' in kwargs:
|
||||
self.source.name = kwargs.pop('file_name')
|
||||
if 'file_hash' in kwargs:
|
||||
@@ -391,8 +395,8 @@ class Collection(BaseClaim):
|
||||
|
||||
def to_dict(self):
|
||||
claim = super().to_dict()
|
||||
if 'claim_references' in claim:
|
||||
claim['claim_references'] = self.claims.ids
|
||||
if claim.pop('claim_references', None):
|
||||
claim['claims'] = self.claims.ids
|
||||
return claim
|
||||
|
||||
@property
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import base64
|
||||
import struct
|
||||
from typing import List
|
||||
from typing import List, Optional, Tuple
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
@@ -33,6 +33,7 @@ class Outputs:
|
||||
txo.meta = {
|
||||
'short_url': f'lbry://{claim.short_url}',
|
||||
'canonical_url': f'lbry://{claim.canonical_url or claim.short_url}',
|
||||
'reposted': claim.reposted,
|
||||
'is_controlling': claim.is_controlling,
|
||||
'take_over_height': claim.take_over_height,
|
||||
'creation_height': claim.creation_height,
|
||||
@@ -47,6 +48,8 @@ class Outputs:
|
||||
}
|
||||
if claim.HasField('channel'):
|
||||
txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
|
||||
if claim.HasField('repost'):
|
||||
txo.reposted_claim = tx_map[claim.repost.tx_hash].outputs[claim.repost.nout]
|
||||
try:
|
||||
if txo.claim.is_channel:
|
||||
txo.meta['claims_in_channel'] = claim.claims_in_channel
|
||||
@@ -80,13 +83,13 @@ class Outputs:
|
||||
if total is not None:
|
||||
page.total = total
|
||||
for row in txo_rows:
|
||||
cls.row_to_message(row, page.txos.add())
|
||||
cls.row_to_message(row, page.txos.add(), extra_txo_rows)
|
||||
for row in extra_txo_rows:
|
||||
cls.row_to_message(row, page.extra_txos.add())
|
||||
cls.row_to_message(row, page.extra_txos.add(), extra_txo_rows)
|
||||
return page.SerializeToString()
|
||||
|
||||
@classmethod
|
||||
def row_to_message(cls, txo, txo_message):
|
||||
def row_to_message(cls, txo, txo_message, extra_txo_rows):
|
||||
if isinstance(txo, Exception):
|
||||
txo_message.error.text = txo.args[0]
|
||||
if isinstance(txo, ValueError):
|
||||
@@ -98,6 +101,7 @@ class Outputs:
|
||||
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
|
||||
txo_message.height = txo['height']
|
||||
txo_message.claim.short_url = txo['short_url']
|
||||
txo_message.claim.reposted = txo['reposted']
|
||||
if txo['canonical_url'] is not None:
|
||||
txo_message.claim.canonical_url = txo['canonical_url']
|
||||
txo_message.claim.is_controlling = bool(txo['is_controlling'])
|
||||
@@ -114,8 +118,16 @@ class Outputs:
|
||||
txo_message.claim.trending_mixed = txo['trending_mixed']
|
||||
txo_message.claim.trending_local = txo['trending_local']
|
||||
txo_message.claim.trending_global = txo['trending_global']
|
||||
if txo['channel_txo_hash']:
|
||||
channel = txo_message.claim.channel
|
||||
channel.tx_hash = txo['channel_txo_hash'][:32]
|
||||
channel.nout, = struct.unpack('<I', txo['channel_txo_hash'][32:])
|
||||
channel.height = txo['channel_height']
|
||||
cls.set_reference(txo_message, 'channel', txo['channel_hash'], extra_txo_rows)
|
||||
cls.set_reference(txo_message, 'repost', txo['reposted_claim_hash'], extra_txo_rows)
|
||||
|
||||
@staticmethod
|
||||
def set_reference(message, attr, claim_hash, rows):
|
||||
if claim_hash:
|
||||
for txo in rows:
|
||||
if claim_hash == txo['claim_hash']:
|
||||
reference = getattr(message.claim, attr)
|
||||
reference.tx_hash = txo['txo_hash'][:32]
|
||||
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
|
||||
reference.height = txo['height']
|
||||
break
|
||||
@@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
package='pb',
|
||||
syntax='proto3',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"b\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\x81\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x02 \x01(\t\x12\x15\n\rcanonical_url\x18\x03 \x01(\t\x12\x16\n\x0eis_controlling\x18\x04 \x01(\x08\x12\x18\n\x10take_over_height\x18\x05 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x06 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x07 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\x08 \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\t \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\n \x01(\x04\x12\x16\n\x0esupport_amount\x18\x0b \x01(\x04\x12\x16\n\x0etrending_group\x18\x0c \x01(\r\x12\x16\n\x0etrending_mixed\x18\r \x01(\x02\x12\x16\n\x0etrending_local\x18\x0e \x01(\x02\x12\x17\n\x0ftrending_global\x18\x0f \x01(\x02\"i\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\"4\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x62\x06proto3')
|
||||
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"b\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"i\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\"4\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x62\x06proto3')
|
||||
)
|
||||
|
||||
|
||||
@@ -45,8 +45,8 @@ _ERROR_CODE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=686,
|
||||
serialized_end=738,
|
||||
serialized_start=732,
|
||||
serialized_end=784,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_ERROR_CODE)
|
||||
|
||||
@@ -180,99 +180,113 @@ _CLAIMMETA = _descriptor.Descriptor(
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='short_url', full_name='pb.ClaimMeta.short_url', index=1,
|
||||
number=2, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
name='repost', full_name='pb.ClaimMeta.repost', index=1,
|
||||
number=2, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=2,
|
||||
name='short_url', full_name='pb.ClaimMeta.short_url', index=2,
|
||||
number=3, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=3,
|
||||
number=4, type=8, cpp_type=7, label=1,
|
||||
name='canonical_url', full_name='pb.ClaimMeta.canonical_url', index=3,
|
||||
number=4, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='is_controlling', full_name='pb.ClaimMeta.is_controlling', index=4,
|
||||
number=5, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='take_over_height', full_name='pb.ClaimMeta.take_over_height', index=4,
|
||||
number=5, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='creation_height', full_name='pb.ClaimMeta.creation_height', index=5,
|
||||
name='take_over_height', full_name='pb.ClaimMeta.take_over_height', index=5,
|
||||
number=6, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='activation_height', full_name='pb.ClaimMeta.activation_height', index=6,
|
||||
name='creation_height', full_name='pb.ClaimMeta.creation_height', index=6,
|
||||
number=7, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=7,
|
||||
name='activation_height', full_name='pb.ClaimMeta.activation_height', index=7,
|
||||
number=8, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=8,
|
||||
name='expiration_height', full_name='pb.ClaimMeta.expiration_height', index=8,
|
||||
number=9, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=9,
|
||||
number=10, type=4, cpp_type=4, label=1,
|
||||
name='claims_in_channel', full_name='pb.ClaimMeta.claims_in_channel', index=9,
|
||||
number=10, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='support_amount', full_name='pb.ClaimMeta.support_amount', index=10,
|
||||
number=11, type=4, cpp_type=4, label=1,
|
||||
name='reposted', full_name='pb.ClaimMeta.reposted', index=10,
|
||||
number=11, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_group', full_name='pb.ClaimMeta.trending_group', index=11,
|
||||
number=12, type=13, cpp_type=3, label=1,
|
||||
name='effective_amount', full_name='pb.ClaimMeta.effective_amount', index=11,
|
||||
number=20, type=4, cpp_type=4, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_mixed', full_name='pb.ClaimMeta.trending_mixed', index=12,
|
||||
number=13, type=2, cpp_type=6, label=1,
|
||||
name='support_amount', full_name='pb.ClaimMeta.support_amount', index=12,
|
||||
number=21, type=4, cpp_type=4, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_group', full_name='pb.ClaimMeta.trending_group', index=13,
|
||||
number=22, type=13, cpp_type=3, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_mixed', full_name='pb.ClaimMeta.trending_mixed', index=14,
|
||||
number=23, type=2, cpp_type=6, label=1,
|
||||
has_default_value=False, default_value=float(0),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_local', full_name='pb.ClaimMeta.trending_local', index=13,
|
||||
number=14, type=2, cpp_type=6, label=1,
|
||||
name='trending_local', full_name='pb.ClaimMeta.trending_local', index=15,
|
||||
number=24, type=2, cpp_type=6, label=1,
|
||||
has_default_value=False, default_value=float(0),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='trending_global', full_name='pb.ClaimMeta.trending_global', index=14,
|
||||
number=15, type=2, cpp_type=6, label=1,
|
||||
name='trending_global', full_name='pb.ClaimMeta.trending_global', index=16,
|
||||
number=25, type=2, cpp_type=6, label=1,
|
||||
has_default_value=False, default_value=float(0),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
@@ -290,7 +304,7 @@ _CLAIMMETA = _descriptor.Descriptor(
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=246,
|
||||
serialized_end=631,
|
||||
serialized_end=677,
|
||||
)
|
||||
|
||||
|
||||
@@ -328,8 +342,8 @@ _ERROR = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=633,
|
||||
serialized_end=738,
|
||||
serialized_start=679,
|
||||
serialized_end=784,
|
||||
)
|
||||
|
||||
_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
|
||||
@@ -343,6 +357,7 @@ _OUTPUT.oneofs_by_name['meta'].fields.append(
|
||||
_OUTPUT.fields_by_name['error'])
|
||||
_OUTPUT.fields_by_name['error'].containing_oneof = _OUTPUT.oneofs_by_name['meta']
|
||||
_CLAIMMETA.fields_by_name['channel'].message_type = _OUTPUT
|
||||
_CLAIMMETA.fields_by_name['repost'].message_type = _OUTPUT
|
||||
_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE
|
||||
_ERROR_CODE.containing_type = _ERROR
|
||||
DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS
|
||||
|
||||
@@ -4,7 +4,7 @@ import logging
|
||||
import binascii
|
||||
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.error import DownloadSDTimeout
|
||||
from lbry.error import DownloadSDTimeoutError
|
||||
from lbry.utils import resolve_host, lru_cache_concurrent
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.blob_exchange.downloader import BlobDownloader
|
||||
@@ -51,7 +51,7 @@ class StreamDownloader:
|
||||
def _delayed_add_fixed_peers():
|
||||
self.added_fixed_peers = True
|
||||
self.peer_queue.put_nowait([
|
||||
make_kademlia_peer(None, address, None, tcp_port=port + 1)
|
||||
make_kademlia_peer(None, address, None, tcp_port=port + 1, allow_localhost=True)
|
||||
for address, port in addresses
|
||||
])
|
||||
|
||||
@@ -82,7 +82,7 @@ class StreamDownloader:
|
||||
log.info("downloaded sd blob %s", self.sd_hash)
|
||||
self.time_to_descriptor = self.loop.time() - now
|
||||
except asyncio.TimeoutError:
|
||||
raise DownloadSDTimeout(self.sd_hash)
|
||||
raise DownloadSDTimeoutError(self.sd_hash)
|
||||
|
||||
# parse the descriptor
|
||||
self.descriptor = await StreamDescriptor.from_stream_descriptor_blob(
|
||||
|
||||
@@ -6,12 +6,12 @@ import logging
|
||||
import binascii
|
||||
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
|
||||
from lbry.utils import generate_id
|
||||
from lbry.error import DownloadSDTimeout
|
||||
from lbry.error import DownloadSDTimeoutError
|
||||
from lbry.schema.mime_types import guess_media_type
|
||||
from lbry.stream.downloader import StreamDownloader
|
||||
from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name
|
||||
from lbry.stream.reflector.client import StreamReflectorClient
|
||||
from lbry.extras.daemon.storage import StoredStreamClaim
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
from lbry.blob import MAX_BLOB_SIZE
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -78,7 +78,7 @@ class ManagedStream:
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredStreamClaim] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredContentClaim] = None,
|
||||
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
|
||||
descriptor: typing.Optional[StreamDescriptor] = None,
|
||||
content_fee: typing.Optional['Transaction'] = None,
|
||||
@@ -253,7 +253,7 @@ class ManagedStream:
|
||||
await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop)
|
||||
except asyncio.TimeoutError:
|
||||
self._running.clear()
|
||||
raise DownloadSDTimeout(self.sd_hash)
|
||||
raise DownloadSDTimeoutError(self.sd_hash)
|
||||
|
||||
if self.delayed_stop_task and not self.delayed_stop_task.done():
|
||||
self.delayed_stop_task.cancel()
|
||||
@@ -404,7 +404,12 @@ class ManagedStream:
|
||||
)
|
||||
await self.update_status(ManagedStream.STATUS_RUNNING)
|
||||
self.file_output_task = self.loop.create_task(self._save_file(self.full_path))
|
||||
await self.started_writing.wait()
|
||||
try:
|
||||
await asyncio.wait_for(self.started_writing.wait(), self.config.download_timeout, loop=self.loop)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("timeout starting to write data for lbry://%s#%s", self.claim_name, self.claim_id)
|
||||
self.stop_tasks()
|
||||
await self.update_status(ManagedStream.STATUS_STOPPED)
|
||||
|
||||
def stop_tasks(self):
|
||||
if self.file_output_task and not self.file_output_task.done():
|
||||
@@ -452,8 +457,8 @@ class ManagedStream:
|
||||
return sent
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
||||
self.stream_claim_info = StoredStreamClaim(
|
||||
self.stream_hash, f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
self.stream_claim_info = StoredContentClaim(
|
||||
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
|
||||
@@ -6,8 +6,8 @@ import random
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.error import ResolveError, InvalidStreamDescriptorError
|
||||
from lbry.error import ResolveTimeout, DownloadDataTimeout
|
||||
from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError
|
||||
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
||||
from lbry.utils import cache_concurrent
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
@@ -20,7 +20,7 @@ if typing.TYPE_CHECKING:
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.dht.node import Node
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredStreamClaim
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||
from lbry.wallet import LbryWalletManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
@@ -117,7 +117,7 @@ class StreamManager:
|
||||
|
||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredStreamClaim'], content_fee: Optional['Transaction'],
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int]):
|
||||
try:
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
|
||||
@@ -371,7 +371,7 @@ class StreamManager:
|
||||
)
|
||||
resolved_result = self._convert_to_old_resolve_output(manager, response)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeout(uri)
|
||||
raise ResolveTimeoutError(uri)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise
|
||||
@@ -437,10 +437,17 @@ class StreamManager:
|
||||
loop=self.loop)
|
||||
return stream
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeout(stream.sd_hash)
|
||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
log.exception("Unexpected error downloading stream:")
|
||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||
KeyFeeAboveMaxAllowedError)
|
||||
if isinstance(err, expected):
|
||||
log.warning("Failed to download %s: %s", uri, str(err))
|
||||
elif isinstance(err, asyncio.CancelledError):
|
||||
pass
|
||||
else:
|
||||
log.exception("Unexpected error downloading stream:")
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
|
||||
@@ -239,6 +239,11 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm
|
||||
)
|
||||
|
||||
def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
|
||||
return self.confirm_and_render(
|
||||
self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def stream_abandon(self, *args, confirm=True, **kwargs):
|
||||
if 'blocking' not in kwargs:
|
||||
kwargs['blocking'] = False
|
||||
@@ -268,6 +273,25 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.daemon.jsonrpc_channel_abandon(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def collection_create(
|
||||
self, name='firstcollection', bid='1.0', confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_collection_create(name, bid, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def collection_update(
|
||||
self, claim_id, confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_collection_update(claim_id, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def collection_abandon(self, *args, confirm=True, **kwargs):
|
||||
if 'blocking' not in kwargs:
|
||||
kwargs['blocking'] = False
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def support_create(self, claim_id, bid='1.0', confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
|
||||
@@ -282,6 +306,9 @@ class CommandTestCase(IntegrationTestCase):
|
||||
async def file_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
|
||||
|
||||
async def claim_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
|
||||
|
||||
@staticmethod
|
||||
def get_claim_id(tx):
|
||||
return tx['outputs'][0]['claim_id']
|
||||
|
||||
@@ -2,8 +2,7 @@ __node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.2.1/lbrycrd-linux.zip'
|
||||
# 'https://github.com/lbryio/lbrycrd/releases/download/v0.17.3.1/lbrycrd-linux-1731.zip'
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.3.2/lbrycrd-linux-1732.zip'
|
||||
)
|
||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from hashlib import sha256
|
||||
from string import hexdigits
|
||||
|
||||
import ecdsa
|
||||
from lbry.wallet.constants import TXO_TYPES
|
||||
from lbry.wallet.constants import CLAIM_TYPES
|
||||
|
||||
from torba.client.baseaccount import BaseAccount, HierarchicalDeterministic
|
||||
|
||||
@@ -91,7 +91,7 @@ class Account(BaseAccount):
|
||||
get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True)
|
||||
total = await get_total_balance()
|
||||
if reserved_subtotals:
|
||||
claims_balance = await get_total_balance(txo_type__in=[TXO_TYPES['stream'], TXO_TYPES['channel']])
|
||||
claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES)
|
||||
for amount, spent, from_me, to_me, height in await self.get_support_summary():
|
||||
if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1):
|
||||
continue
|
||||
@@ -163,6 +163,12 @@ class Account(BaseAccount):
|
||||
def get_channel_count(self, **constraints):
|
||||
return self.ledger.get_channel_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_collections(self, **constraints):
|
||||
return self.ledger.get_collections(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_collection_count(self, **constraints):
|
||||
return self.ledger.get_collection_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_supports(self, **constraints):
|
||||
return self.ledger.get_supports(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
|
||||
@@ -2,5 +2,14 @@ TXO_TYPES = {
|
||||
"stream": 1,
|
||||
"channel": 2,
|
||||
"support": 3,
|
||||
"purchase": 4
|
||||
"purchase": 4,
|
||||
"collection": 5,
|
||||
"repost": 6,
|
||||
}
|
||||
|
||||
CLAIM_TYPES = [
|
||||
TXO_TYPES['stream'],
|
||||
TXO_TYPES['channel'],
|
||||
TXO_TYPES['collection'],
|
||||
TXO_TYPES['repost'],
|
||||
]
|
||||
@@ -3,7 +3,7 @@ from typing import List
|
||||
from torba.client.basedatabase import BaseDatabase
|
||||
|
||||
from lbry.wallet.transaction import Output
|
||||
from lbry.wallet.constants import TXO_TYPES
|
||||
from lbry.wallet.constants import TXO_TYPES, CLAIM_TYPES
|
||||
|
||||
|
||||
class WalletDatabase(BaseDatabase):
|
||||
@@ -140,9 +140,11 @@ class WalletDatabase(BaseDatabase):
|
||||
|
||||
@staticmethod
|
||||
def constrain_claims(constraints):
|
||||
constraints['txo_type__in'] = [
|
||||
TXO_TYPES['stream'], TXO_TYPES['channel']
|
||||
]
|
||||
claim_type = constraints.pop('claim_type', None)
|
||||
if claim_type is not None:
|
||||
constraints['txo_type'] = TXO_TYPES[claim_type]
|
||||
else:
|
||||
constraints['txo_type__in'] = CLAIM_TYPES
|
||||
|
||||
async def get_claims(self, **constraints) -> List[Output]:
|
||||
self.constrain_claims(constraints)
|
||||
@@ -188,6 +190,18 @@ class WalletDatabase(BaseDatabase):
|
||||
self.constrain_supports(constraints)
|
||||
return self.get_utxo_count(**constraints)
|
||||
|
||||
@staticmethod
|
||||
def constrain_collections(constraints):
|
||||
constraints['txo_type'] = TXO_TYPES['collection']
|
||||
|
||||
def get_collections(self, **constraints):
|
||||
self.constrain_collections(constraints)
|
||||
return self.get_utxos(**constraints)
|
||||
|
||||
def get_collection_count(self, **constraints):
|
||||
self.constrain_collections(constraints)
|
||||
return self.get_utxo_count(**constraints)
|
||||
|
||||
async def release_all_outputs(self, account):
|
||||
await self.db.execute_fetchall(
|
||||
"UPDATE txo SET is_reserved = 0 WHERE"
|
||||
|
||||
@@ -15,6 +15,7 @@ class Headers(BaseHeaders):
|
||||
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
|
||||
target_timespan = 150
|
||||
checkpoint = (600_000, b'100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2')
|
||||
|
||||
@property
|
||||
def claim_trie_root(self):
|
||||
|
||||
@@ -186,6 +186,35 @@ class MainNetLedger(BaseLedger):
|
||||
def get_channel_count(self, **constraints):
|
||||
return self.db.get_channel_count(**constraints)
|
||||
|
||||
async def resolve_collection(self, collection, offset=0, page_size=1):
|
||||
claim_ids = collection.claim.collection.claims.ids[offset:page_size+offset]
|
||||
try:
|
||||
resolve_results, _, _ = await self.claim_search([], claim_ids=claim_ids)
|
||||
except:
|
||||
log.exception("Resolve failed while looking up collection claim ids:")
|
||||
return []
|
||||
claims = []
|
||||
for claim_id in claim_ids:
|
||||
found = False
|
||||
for txo in resolve_results:
|
||||
if txo.claim_id == claim_id:
|
||||
claims.append(txo)
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
claims.append(None)
|
||||
return claims
|
||||
|
||||
async def get_collections(self, resolve_claims=0, **constraints):
|
||||
collections = await self.db.get_collections(**constraints)
|
||||
if resolve_claims > 0:
|
||||
for collection in collections:
|
||||
collection.claims = await self.resolve_collection(collection, page_size=resolve_claims)
|
||||
return collections
|
||||
|
||||
def get_collection_count(self, resolve_claims=0, **constraints):
|
||||
return self.db.get_collection_count(**constraints)
|
||||
|
||||
def get_supports(self, **constraints):
|
||||
return self.db.get_supports(**constraints)
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from torba.client.basemanager import BaseWalletManager
|
||||
from torba.client.wallet import ENCRYPT_ON_DISK
|
||||
from torba.rpc.jsonrpc import CodeMessageError
|
||||
|
||||
from lbry.error import KeyFeeAboveMaxAllowed
|
||||
from lbry.error import KeyFeeAboveMaxAllowedError
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet.account import Account
|
||||
from lbry.wallet.ledger import MainNetLedger
|
||||
@@ -176,10 +176,10 @@ class LbryWalletManager(BaseWalletManager):
|
||||
if not tx:
|
||||
try:
|
||||
raw = await self.ledger.network.get_transaction(txid)
|
||||
if not raw:
|
||||
return {'success': False, 'code': 404, 'message': 'transaction not found'}
|
||||
height = await self.ledger.network.get_transaction_height(txid)
|
||||
except CodeMessageError as e:
|
||||
if 'No such mempool or blockchain transaction.' in e.message:
|
||||
return {'success': False, 'code': 404, 'message': 'transaction not found'}
|
||||
return {'success': False, 'code': e.code, 'message': e.message}
|
||||
tx = self.ledger.transaction_class(unhexlify(raw))
|
||||
await self.ledger.maybe_verify_transaction(tx, height)
|
||||
@@ -199,7 +199,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
error_max_fee = f"{dewies_to_lbc(max_fee_amount)} LBC"
|
||||
if max_fee['currency'] != 'LBC':
|
||||
error_max_fee += f" ({max_fee['amount']} {max_fee['currency']})"
|
||||
raise KeyFeeAboveMaxAllowed(
|
||||
raise KeyFeeAboveMaxAllowedError(
|
||||
f"Purchase price of {error_fee} exceeds maximum "
|
||||
f"configured price of {error_max_fee}."
|
||||
)
|
||||
|
||||
@@ -2,6 +2,7 @@ from torba.client.basenetwork import BaseNetwork
|
||||
|
||||
|
||||
class Network(BaseNetwork):
|
||||
PROTOCOL_VERSION = '2.0'
|
||||
|
||||
def get_claims_by_ids(self, claim_ids):
|
||||
return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids)
|
||||
|
||||
@@ -5,6 +5,7 @@ from torba.server.script import ScriptPubKey, OpCodes
|
||||
from torba.server.util import cachedproperty
|
||||
from torba.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from torba.server.coins import Coin, CoinError
|
||||
from torba.server.tx import DeserializerSegWit
|
||||
|
||||
from lbry.wallet.script import OutputScript
|
||||
from .session import LBRYElectrumX, LBRYSessionManager
|
||||
@@ -18,6 +19,7 @@ class LBC(Coin):
|
||||
SESSIONCLS = LBRYElectrumX
|
||||
BLOCK_PROCESSOR = LBRYBlockProcessor
|
||||
SESSION_MANAGER = LBRYSessionManager
|
||||
DESERIALIZER = DeserializerSegWit
|
||||
DB = LBRYDB
|
||||
NAME = "LBRY"
|
||||
SHORTNAME = "LBC"
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import sqlite3
|
||||
from torba.client.basedatabase import constraints_to_sql
|
||||
|
||||
CREATE_FULL_TEXT_SEARCH = """
|
||||
create virtual table if not exists search using fts5(
|
||||
claim_name, channel_name, title, description, author, tags,
|
||||
content=claim, tokenize=porter
|
||||
);
|
||||
"""
|
||||
|
||||
FTS_ORDER_BY = "bm25(search, 4.0, 8.0, 1.0, 0.5, 1.0, 0.5)"
|
||||
|
||||
|
||||
def fts_action_sql(claims=None, action='insert'):
|
||||
select = {
|
||||
'rowid': "claim.rowid",
|
||||
'claim_name': "claim.normalized",
|
||||
'channel_name': "channel.normalized",
|
||||
'title': "claim.title",
|
||||
'description': "claim.description",
|
||||
'author': "claim.author",
|
||||
'tags': "(select group_concat(tag, ' ') from tag where tag.claim_hash=claim.claim_hash)"
|
||||
}
|
||||
if action == 'delete':
|
||||
select['search'] = '"delete"'
|
||||
|
||||
where, values = "", {}
|
||||
if claims:
|
||||
where, values = constraints_to_sql({
|
||||
'claim.claim_hash__in': [sqlite3.Binary(claim_hash) for claim_hash in claims]
|
||||
})
|
||||
where = 'WHERE '+where
|
||||
|
||||
return f"""
|
||||
INSERT INTO search ({','.join(select.keys())})
|
||||
SELECT {','.join(select.values())} FROM claim
|
||||
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash) {where}
|
||||
""", values
|
||||
|
||||
|
||||
def update_full_text_search(action, outputs, db, is_first_sync):
|
||||
if is_first_sync:
|
||||
return
|
||||
if not outputs:
|
||||
return
|
||||
if action in ("before-delete", "before-update"):
|
||||
db.execute(*fts_action_sql(outputs, 'delete'))
|
||||
elif action in ("after-insert", "after-update"):
|
||||
db.execute(*fts_action_sql(outputs, 'insert'))
|
||||
else:
|
||||
raise ValueError(f"Invalid action for updating full text search: '{action}'")
|
||||
|
||||
|
||||
def first_sync_finished(db):
|
||||
db.execute(*fts_action_sql())
|
||||
@@ -18,6 +18,7 @@ from lbry.schema.result import Outputs
|
||||
from lbry.wallet.ledger import BaseLedger, MainNetLedger, RegTestLedger
|
||||
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
from .full_text_search import FTS_ORDER_BY
|
||||
|
||||
|
||||
class SQLiteOperationalError(sqlite3.OperationalError):
|
||||
@@ -33,22 +34,22 @@ class SQLiteInterruptedError(sqlite3.OperationalError):
|
||||
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
|
||||
sqlite3.enable_callback_tracebacks(True)
|
||||
|
||||
INTEGER_PARAMS = {
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'timestamp', 'creation_timestamp', 'release_time', 'fee_amount',
|
||||
'tx_position', 'channel_join',
|
||||
'tx_position', 'channel_join', 'reposted',
|
||||
'amount', 'effective_amount', 'support_amount',
|
||||
'trending_group', 'trending_mixed',
|
||||
'trending_local', 'trending_global',
|
||||
}
|
||||
|
||||
SEARCH_PARAMS = {
|
||||
'name', 'claim_id', 'claim_ids', 'txid', 'nout', 'channel', 'channel_ids', 'not_channel_ids',
|
||||
'name', 'text', 'claim_id', 'claim_ids', 'txid', 'nout', 'channel', 'channel_ids', 'not_channel_ids',
|
||||
'public_key_id', 'claim_type', 'stream_types', 'media_types', 'fee_currency',
|
||||
'has_channel_signature', 'signature_valid',
|
||||
'any_tags', 'all_tags', 'not_tags',
|
||||
'has_channel_signature', 'signature_valid', 'blocklist_channel_ids',
|
||||
'any_tags', 'all_tags', 'not_tags', 'reposted_claim_id',
|
||||
'any_locations', 'all_locations', 'not_locations',
|
||||
'any_languages', 'all_languages', 'not_languages',
|
||||
'is_controlling', 'limit', 'offset', 'order_by',
|
||||
@@ -229,6 +230,9 @@ def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
elif 'claim_ids' in constraints:
|
||||
constraints['claim.claim_id__in'] = constraints.pop('claim_ids')
|
||||
|
||||
if 'reposted_claim_id' in constraints:
|
||||
constraints['claim.reposted_claim_hash'] = sqlite3.Binary(unhexlify(constraints.pop('reposted_claim_id'))[::-1])
|
||||
|
||||
if 'name' in constraints:
|
||||
constraints['claim.normalized'] = normalize_name(constraints.pop('name'))
|
||||
|
||||
@@ -256,6 +260,21 @@ def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
'claim.signature_valid__is_null': True,
|
||||
'claim.channel_hash__not_in': not_channel_ids_binary
|
||||
}
|
||||
if 'blocklist_channel_ids' in constraints:
|
||||
blocklist_ids = constraints.pop('blocklist_channel_ids')
|
||||
if blocklist_ids:
|
||||
blocking_channels = [
|
||||
sqlite3.Binary(unhexlify(channel_id)[::-1]) for channel_id in blocklist_ids
|
||||
]
|
||||
constraints.update({
|
||||
f'$blocking_channel{i}': a for i, a in enumerate(blocking_channels)
|
||||
})
|
||||
blocklist = ', '.join([
|
||||
f':$blocking_channel{i}' for i in range(len(blocking_channels))
|
||||
])
|
||||
constraints['claim.claim_hash__not_in#blocklist_channel_ids'] = f"""
|
||||
SELECT reposted_claim_hash FROM claim WHERE channel_hash IN ({blocklist})
|
||||
"""
|
||||
if 'signature_valid' in constraints:
|
||||
has_channel_signature = constraints.pop('has_channel_signature', False)
|
||||
if has_channel_signature:
|
||||
@@ -295,15 +314,15 @@ def _get_claims(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
_apply_constraints_for_array_attributes(constraints, 'language', lambda _: _, for_count)
|
||||
_apply_constraints_for_array_attributes(constraints, 'location', lambda _: _, for_count)
|
||||
|
||||
select = f"SELECT {cols} FROM claim"
|
||||
|
||||
sql, values = query(
|
||||
select if for_count else select+"""
|
||||
LEFT JOIN claimtrie USING (claim_hash)
|
||||
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash)
|
||||
""", **constraints
|
||||
)
|
||||
return sql, values
|
||||
if 'text' in constraints:
|
||||
constraints["search"] = constraints.pop("text")
|
||||
constraints["order_by"] = FTS_ORDER_BY
|
||||
select = f"SELECT {cols} FROM search JOIN claim ON (search.rowid=claim.rowid)"
|
||||
else:
|
||||
select = f"SELECT {cols} FROM claim"
|
||||
if not for_count:
|
||||
select += " LEFT JOIN claimtrie USING (claim_hash)"
|
||||
return query(select, **constraints)
|
||||
|
||||
|
||||
def get_claims(cols, for_count=False, **constraints) -> List:
|
||||
@@ -327,6 +346,46 @@ def get_claims_count(**constraints) -> int:
|
||||
return count[0][0]
|
||||
|
||||
|
||||
def _search(**constraints):
|
||||
return get_claims(
|
||||
"""
|
||||
claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
claim.claim_hash, claim.txo_hash,
|
||||
claim.claims_in_channel, claim.reposted,
|
||||
claim.height, claim.creation_height,
|
||||
claim.activation_height, claim.expiration_height,
|
||||
claim.effective_amount, claim.support_amount,
|
||||
claim.trending_group, claim.trending_mixed,
|
||||
claim.trending_local, claim.trending_global,
|
||||
claim.short_url, claim.canonical_url,
|
||||
claim.channel_hash, claim.reposted_claim_hash,
|
||||
claim.signature_valid
|
||||
""", **constraints
|
||||
)
|
||||
|
||||
|
||||
def _get_referenced_rows(txo_rows: List[sqlite3.Row]):
|
||||
repost_hashes = set(filter(None, map(itemgetter('reposted_claim_hash'), txo_rows)))
|
||||
channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows)))
|
||||
|
||||
reposted_txos = []
|
||||
if repost_hashes:
|
||||
reposted_txos = _search(
|
||||
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in repost_hashes]}
|
||||
)
|
||||
channel_hashes |= set(filter(None, map(itemgetter('channel_hash'), reposted_txos)))
|
||||
|
||||
channel_txos = []
|
||||
if channel_hashes:
|
||||
channel_txos = _search(
|
||||
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}
|
||||
)
|
||||
|
||||
# channels must come first for client side inflation to work properly
|
||||
return channel_txos + reposted_txos
|
||||
|
||||
|
||||
@measure
|
||||
def search(constraints) -> Tuple[List, List, int, int]:
|
||||
assert set(constraints).issubset(SEARCH_PARAMS), \
|
||||
@@ -339,49 +398,15 @@ def search(constraints) -> Tuple[List, List, int, int]:
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = ["claim_hash"]
|
||||
txo_rows = _search(**constraints)
|
||||
channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows)))
|
||||
extra_txo_rows = []
|
||||
if channel_hashes:
|
||||
extra_txo_rows = _search(
|
||||
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}
|
||||
)
|
||||
extra_txo_rows = _get_referenced_rows(txo_rows)
|
||||
return txo_rows, extra_txo_rows, constraints['offset'], total
|
||||
|
||||
|
||||
def _search(**constraints):
|
||||
return get_claims(
|
||||
"""
|
||||
claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
claim.claim_hash, claim.txo_hash,
|
||||
claim.claims_in_channel,
|
||||
claim.height, claim.creation_height,
|
||||
claim.activation_height, claim.expiration_height,
|
||||
claim.effective_amount, claim.support_amount,
|
||||
claim.trending_group, claim.trending_mixed,
|
||||
claim.trending_local, claim.trending_global,
|
||||
claim.short_url, claim.canonical_url,
|
||||
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
|
||||
channel.height AS channel_height, claim.signature_valid
|
||||
""", **constraints
|
||||
)
|
||||
|
||||
|
||||
@measure
|
||||
def resolve(urls) -> Tuple[List, List]:
|
||||
result = []
|
||||
channel_hashes = set()
|
||||
for raw_url in urls:
|
||||
match = resolve_url(raw_url)
|
||||
result.append(match)
|
||||
if isinstance(match, sqlite3.Row) and match['channel_hash']:
|
||||
channel_hashes.add(match['channel_hash'])
|
||||
extra_txo_rows = []
|
||||
if channel_hashes:
|
||||
extra_txo_rows = _search(
|
||||
**{'claim.claim_hash__in': [sqlite3.Binary(h) for h in channel_hashes]}
|
||||
)
|
||||
return result, extra_txo_rows
|
||||
txo_rows = [resolve_url(raw_url) for raw_url in urls]
|
||||
extra_txo_rows = _get_referenced_rows([r for r in txo_rows if isinstance(r, sqlite3.Row)])
|
||||
return txo_rows, extra_txo_rows
|
||||
|
||||
|
||||
@measure
|
||||
|
||||
@@ -39,7 +39,8 @@ class ZScore:
|
||||
|
||||
@property
|
||||
def standard_deviation(self):
|
||||
return sqrt((self.power / self.count) - self.mean ** 2)
|
||||
value = (self.power / self.count) - self.mean ** 2
|
||||
return sqrt(value) if value > 0 else 0
|
||||
|
||||
def finalize(self):
|
||||
if self.count == 0:
|
||||
|
||||
@@ -13,6 +13,7 @@ from lbry.schema.mime_types import guess_stream_type
|
||||
from lbry.wallet.ledger import MainNetLedger, RegTestLedger
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
from lbry.wallet.server.db.canonical import register_canonical_functions
|
||||
from lbry.wallet.server.db.full_text_search import update_full_text_search, CREATE_FULL_TEXT_SEARCH, first_sync_finished
|
||||
from lbry.wallet.server.db.trending import (
|
||||
CREATE_TREND_TABLE, calculate_trending, register_trending_functions
|
||||
)
|
||||
@@ -21,6 +22,7 @@ from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
sqlite3.enable_callback_tracebacks(True)
|
||||
|
||||
|
||||
class SQLDB:
|
||||
@@ -49,7 +51,12 @@ class SQLDB:
|
||||
short_url text not null, -- normalized#shortest-unique-claim_id
|
||||
canonical_url text, -- channel's-short_url/normalized#shortest-unique-claim_id-within-channel
|
||||
|
||||
title text,
|
||||
author text,
|
||||
description text,
|
||||
|
||||
claim_type integer,
|
||||
reposted integer default 0,
|
||||
|
||||
-- streams
|
||||
stream_type text,
|
||||
@@ -57,6 +64,9 @@ class SQLDB:
|
||||
fee_amount integer default 0,
|
||||
fee_currency text,
|
||||
|
||||
-- reposts
|
||||
reposted_claim_hash bytes,
|
||||
|
||||
-- claims which are channels
|
||||
public_key_bytes bytes,
|
||||
public_key_hash bytes,
|
||||
@@ -151,6 +161,7 @@ class SQLDB:
|
||||
PRAGMAS +
|
||||
CREATE_CLAIM_TABLE +
|
||||
CREATE_TREND_TABLE +
|
||||
CREATE_FULL_TEXT_SEARCH +
|
||||
CREATE_SUPPORT_TABLE +
|
||||
CREATE_CLAIMTRIE_TABLE +
|
||||
CREATE_TAG_TABLE
|
||||
@@ -162,6 +173,7 @@ class SQLDB:
|
||||
self.db = None
|
||||
self.logger = class_logger(__name__, self.__class__.__name__)
|
||||
self.ledger = MainNetLedger if self.main.coin.NET == 'mainnet' else RegTestLedger
|
||||
self._fts_synced = False
|
||||
|
||||
def open(self):
|
||||
self.db = sqlite3.connect(self._db_path, isolation_level=None, check_same_thread=False, uri=True)
|
||||
@@ -233,12 +245,16 @@ class SQLDB:
|
||||
'amount': txo.amount,
|
||||
'timestamp': header['timestamp'],
|
||||
'height': tx.height,
|
||||
'title': None,
|
||||
'description': None,
|
||||
'author': None,
|
||||
'claim_type': None,
|
||||
'stream_type': None,
|
||||
'media_type': None,
|
||||
'release_time': None,
|
||||
'fee_currency': None,
|
||||
'fee_amount': 0
|
||||
'fee_amount': 0,
|
||||
'reposted_claim_hash': None
|
||||
}
|
||||
claims.append(claim_record)
|
||||
|
||||
@@ -252,6 +268,9 @@ class SQLDB:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['stream']
|
||||
claim_record['media_type'] = claim.stream.source.media_type
|
||||
claim_record['stream_type'] = STREAM_TYPES[guess_stream_type(claim_record['media_type'])]
|
||||
claim_record['title'] = claim.stream.title
|
||||
claim_record['description'] = claim.stream.description
|
||||
claim_record['author'] = claim.stream.author
|
||||
if claim.stream.release_time:
|
||||
claim_record['release_time'] = claim.stream.release_time
|
||||
if claim.stream.has_fee:
|
||||
@@ -260,6 +279,8 @@ class SQLDB:
|
||||
claim_record['fee_currency'] = fee.currency.lower()
|
||||
if isinstance(fee.amount, Decimal):
|
||||
claim_record['fee_amount'] = int(fee.amount*1000)
|
||||
elif claim.is_repost:
|
||||
claim_record['reposted_claim_hash'] = claim.repost.reference.claim_hash
|
||||
elif claim.is_channel:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['channel']
|
||||
|
||||
@@ -283,12 +304,12 @@ class SQLDB:
|
||||
INSERT OR IGNORE INTO claim (
|
||||
claim_hash, claim_id, claim_name, normalized, txo_hash, tx_position, amount,
|
||||
claim_type, media_type, stream_type, timestamp, creation_timestamp,
|
||||
fee_currency, fee_amount, height,
|
||||
fee_currency, fee_amount, title, description, author, height, reposted_claim_hash,
|
||||
creation_height, release_time, activation_height, expiration_height, short_url)
|
||||
VALUES (
|
||||
:claim_hash, :claim_id, :claim_name, :normalized, :txo_hash, :tx_position, :amount,
|
||||
:claim_type, :media_type, :stream_type, :timestamp, :timestamp,
|
||||
:fee_currency, :fee_amount, :height, :height,
|
||||
:fee_currency, :fee_amount, :title, :description, :author, :height, :reposted_claim_hash, :height,
|
||||
CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE :timestamp END,
|
||||
CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END,
|
||||
CASE WHEN :height >= 137181 THEN :height+2102400 ELSE :height+262974 END,
|
||||
@@ -306,6 +327,7 @@ class SQLDB:
|
||||
txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height,
|
||||
claim_type=:claim_type, media_type=:media_type, stream_type=:stream_type,
|
||||
timestamp=:timestamp, fee_amount=:fee_amount, fee_currency=:fee_currency,
|
||||
title=:title, description=:description, author=:author, reposted_claim_hash=:reposted_claim_hash,
|
||||
release_time=CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE release_time END
|
||||
WHERE claim_hash=:claim_hash;
|
||||
""", claims)
|
||||
@@ -366,6 +388,25 @@ class SQLDB:
|
||||
'support', {'txo_hash__in': [sqlite3.Binary(txo_hash) for txo_hash in txo_hashes]}
|
||||
))
|
||||
|
||||
def calculate_reposts(self, txos: List[Output]):
|
||||
targets = set()
|
||||
for txo in txos:
|
||||
try:
|
||||
claim = txo.claim
|
||||
except:
|
||||
continue
|
||||
if claim.is_repost:
|
||||
targets.add((claim.repost.reference.claim_hash,))
|
||||
if targets:
|
||||
self.db.executemany(
|
||||
"""
|
||||
UPDATE claim SET reposted = (
|
||||
SELECT count(*) FROM claim AS repost WHERE repost.reposted_claim_hash = claim.claim_hash
|
||||
)
|
||||
WHERE claim_hash = ?
|
||||
""", targets
|
||||
)
|
||||
|
||||
def validate_channel_signatures(self, height, new_claims, updated_claims, spent_claims, affected_channels, timer):
|
||||
if not new_claims and not updated_claims and not spent_claims:
|
||||
return
|
||||
@@ -649,7 +690,7 @@ class SQLDB:
|
||||
body_timer = timer.add_timer('body')
|
||||
for position, (etx, txid) in enumerate(all_txs):
|
||||
tx = timer.run(
|
||||
Transaction, etx.serialize(), height=height, position=position
|
||||
Transaction, etx.raw, height=height, position=position
|
||||
)
|
||||
# Inputs
|
||||
spent_claims, spent_supports, spent_others = timer.run(
|
||||
@@ -692,15 +733,27 @@ class SQLDB:
|
||||
expire_timer.stop()
|
||||
|
||||
r = timer.run
|
||||
r(update_full_text_search, 'before-delete',
|
||||
delete_claim_hashes, self.db, self.main.first_sync)
|
||||
affected_channels = r(self.delete_claims, delete_claim_hashes)
|
||||
r(self.delete_supports, delete_support_txo_hashes)
|
||||
r(self.insert_claims, insert_claims, header)
|
||||
r(self.calculate_reposts, insert_claims)
|
||||
r(update_full_text_search, 'after-insert',
|
||||
[txo.claim_hash for txo in insert_claims], self.db, self.main.first_sync)
|
||||
r(update_full_text_search, 'before-update',
|
||||
[txo.claim_hash for txo in update_claims], self.db, self.main.first_sync)
|
||||
r(self.update_claims, update_claims, header)
|
||||
r(update_full_text_search, 'after-update',
|
||||
[txo.claim_hash for txo in update_claims], self.db, self.main.first_sync)
|
||||
r(self.validate_channel_signatures, height, insert_claims,
|
||||
update_claims, delete_claim_hashes, affected_channels, forward_timer=True)
|
||||
r(self.insert_supports, insert_supports)
|
||||
r(self.update_claimtrie, height, recalculate_claim_hashes, deleted_claim_names, forward_timer=True)
|
||||
r(calculate_trending, self.db, height, daemon_height)
|
||||
if not self._fts_synced and self.main.first_sync and height == daemon_height:
|
||||
r(first_sync_finished, self.db)
|
||||
self._fts_synced = True
|
||||
|
||||
|
||||
class LBRYDB(DB):
|
||||
|
||||
@@ -3,6 +3,7 @@ import math
|
||||
import time
|
||||
import base64
|
||||
import asyncio
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
from pylru import lrucache
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
||||
@@ -18,6 +19,8 @@ from lbry.wallet.server.websocket import AdminWebSocket
|
||||
from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
|
||||
from lbry import __version__ as sdk_version
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ResultCacheItem:
|
||||
__slots__ = '_result', 'lock', 'has_result'
|
||||
@@ -92,7 +95,8 @@ class LBRYSessionManager(SessionManager):
|
||||
|
||||
|
||||
class LBRYElectrumX(ElectrumX):
|
||||
PROTOCOL_MIN = (0, 0) # temporary, for supporting 0.10 protocol
|
||||
PROTOCOL_MIN = (2, 0) # v0.48 is backwards incompatible, forking a new protocol
|
||||
PROTOCOL_MAX = (2, 0)
|
||||
max_errors = math.inf # don't disconnect people for errors! let them happen...
|
||||
session_mgr: LBRYSessionManager
|
||||
version = sdk_version
|
||||
@@ -103,9 +107,12 @@ class LBRYElectrumX(ElectrumX):
|
||||
self.daemon = self.session_mgr.daemon
|
||||
self.bp: LBRYBlockProcessor = self.session_mgr.bp
|
||||
self.db: LBRYDB = self.bp.db
|
||||
# space separated list of channel URIs used for filtering bad content
|
||||
filtering_channels = self.env.default('FILTERING_CHANNELS_IDS', '1111')
|
||||
self.filtering_channels_ids = list(filter(None, filtering_channels.split(' ')))
|
||||
|
||||
def set_request_handlers(self, ptuple):
|
||||
super().set_request_handlers(ptuple)
|
||||
super().set_request_handlers((1, 2) if ptuple == (2, 0) else ptuple) # our "2.0" is electrumx 1.2
|
||||
handlers = {
|
||||
'blockchain.transaction.get_height': self.transaction_get_height,
|
||||
'blockchain.claimtrie.search': self.claimtrie_search,
|
||||
@@ -131,6 +138,8 @@ class LBRYElectrumX(ElectrumX):
|
||||
result = await asyncio.get_running_loop().run_in_executor(
|
||||
self.session_mgr.query_executor, func, kwargs
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except reader.SQLiteInterruptedError as error:
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_interrupt(start, error.metrics)
|
||||
@@ -139,7 +148,8 @@ class LBRYElectrumX(ElectrumX):
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_error(start, error.metrics)
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute')
|
||||
except:
|
||||
except Exception:
|
||||
log.exception("dear devs, please handle this exception better")
|
||||
metrics = self.get_metrics_or_placeholder_for_api(query_name)
|
||||
metrics.query_error(start, {})
|
||||
raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error')
|
||||
@@ -174,6 +184,7 @@ class LBRYElectrumX(ElectrumX):
|
||||
|
||||
async def claimtrie_search(self, **kwargs):
|
||||
if kwargs:
|
||||
kwargs['blocklist_channel_ids'] = self.filtering_channels_ids
|
||||
return await self.run_and_cache_query('search', reader.search_to_bytes, kwargs)
|
||||
|
||||
async def claimtrie_resolve(self, *urls):
|
||||
|
||||
@@ -32,6 +32,7 @@ class Output(BaseOutput):
|
||||
__slots__ = (
|
||||
'channel', 'private_key', 'meta',
|
||||
'purchase', 'purchased_claim', 'purchase_receipt',
|
||||
'reposted_claim', 'claims',
|
||||
)
|
||||
|
||||
def __init__(self, *args, channel: Optional['Output'] = None,
|
||||
@@ -42,6 +43,8 @@ class Output(BaseOutput):
|
||||
self.purchase: 'Output' = None # txo containing purchase metadata
|
||||
self.purchased_claim: 'Output' = None # resolved claim pointed to by purchase
|
||||
self.purchase_receipt: 'Output' = None # txo representing purchase receipt for this claim
|
||||
self.reposted_claim: 'Output' = None # txo representing claim being reposted
|
||||
self.claims: List['Output'] = None # resolved claims for collection
|
||||
self.meta = {}
|
||||
|
||||
def update_annotations(self, annotated):
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
"""Set the build version to be 'qa', 'rc', 'release'"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
|
||||
log = logging.getLogger()
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def get_build_type(travis_tag=None):
|
||||
if not travis_tag:
|
||||
return "qa"
|
||||
log.debug("getting build type for tag: \"%s\"", travis_tag)
|
||||
if re.match(r'v\d+\.\d+\.\d+rc\d+$', travis_tag):
|
||||
return 'rc'
|
||||
elif re.match(r'v\d+\.\d+\.\d+$', travis_tag):
|
||||
return 'release'
|
||||
return 'qa'
|
||||
|
||||
|
||||
def main():
|
||||
root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
build_type_path = os.path.join(root_dir, 'lbry', 'build_type.py')
|
||||
log.debug("configuring build type file: %s", build_type_path)
|
||||
travis_commit = os.environ['TRAVIS_COMMIT'][:6]
|
||||
build_type = get_build_type(os.environ.get('TRAVIS_TAG', None))
|
||||
log.debug("setting build type=%s, build commit=%s", build_type, travis_commit)
|
||||
with open(build_type_path, 'w') as f:
|
||||
f.write(f"BUILD = \"{build_type}\"\nBUILD_COMMIT = \"{travis_commit}\"\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -92,7 +92,7 @@ async def monitor(db, server):
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(10)) as session:
|
||||
try:
|
||||
ws = await session.ws_connect(server)
|
||||
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
|
||||
except (aiohttp.ClientConnectionError, asyncio.TimeoutError):
|
||||
if first_attempt:
|
||||
print(f"failed connecting to {server}")
|
||||
await boris_says(random.choice([
|
||||
@@ -130,7 +130,7 @@ async def monitor(db, server):
|
||||
await handle_analytics_event(c, event, server)
|
||||
db.commit()
|
||||
|
||||
except (aiohttp.ClientConnectorError, asyncio.TimeoutError):
|
||||
except (aiohttp.ClientConnectionError, asyncio.TimeoutError):
|
||||
await boris_says(random.choice([
|
||||
f"<!channel> Guys, we have a problem! Nobody home at {server}. Will check on it again in {delay} seconds.",
|
||||
f"<!channel> Something wrong with {server}. I think dead. Will poke it again in {delay} seconds.",
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ setup(
|
||||
install_requires=[
|
||||
'torba',
|
||||
'aiohttp==3.5.4',
|
||||
'aioupnp==0.0.15',
|
||||
'aioupnp==0.0.16',
|
||||
'appdirs==1.4.3',
|
||||
'certifi>=2018.11.29',
|
||||
'colorama==0.3.7',
|
||||
|
||||
@@ -48,8 +48,9 @@ def get_time_accelerator(loop: asyncio.AbstractEventLoop,
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def mock_network_loop(loop: asyncio.AbstractEventLoop):
|
||||
dht_network: typing.Dict[typing.Tuple[str, int], 'KademliaProtocol'] = {}
|
||||
def mock_network_loop(loop: asyncio.AbstractEventLoop,
|
||||
dht_network: typing.Optional[typing.Dict[typing.Tuple[str, int], 'KademliaProtocol']] = None):
|
||||
dht_network: typing.Dict[typing.Tuple[str, int], 'KademliaProtocol'] = dht_network if dht_network is not None else {}
|
||||
|
||||
async def create_datagram_endpoint(proto_lam: typing.Callable[[], 'KademliaProtocol'],
|
||||
from_addr: typing.Tuple[str, int]):
|
||||
|
||||
@@ -328,6 +328,52 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.assertFindsClaims([image], media_types=['image/png'])
|
||||
await self.assertFindsClaims([image, video], media_types=['video/mp4', 'image/png'])
|
||||
|
||||
async def test_search_by_text(self):
|
||||
chan1_id = self.get_claim_id(await self.channel_create('@SatoshiNakamoto'))
|
||||
chan2_id = self.get_claim_id(await self.channel_create('@Bitcoin'))
|
||||
chan3_id = self.get_claim_id(await self.channel_create('@IAmSatoshi'))
|
||||
|
||||
claim1 = await self.stream_create(
|
||||
"the-real-satoshi", title="The Real Satoshi Nakamoto",
|
||||
description="Documentary about the real Satoshi Nakamoto, creator of bitcoin.",
|
||||
tags=['satoshi nakamoto', 'bitcoin', 'documentary']
|
||||
)
|
||||
claim2 = await self.stream_create(
|
||||
"about-me", channel_id=chan1_id, title="Satoshi Nakamoto Autobiography",
|
||||
description="I am Satoshi Nakamoto and this is my autobiography.",
|
||||
tags=['satoshi nakamoto', 'bitcoin', 'documentary', 'autobiography']
|
||||
)
|
||||
claim3 = await self.stream_create(
|
||||
"history-of-bitcoin", channel_id=chan2_id, title="History of Bitcoin",
|
||||
description="History of bitcoin and its creator Satoshi Nakamoto.",
|
||||
tags=['satoshi nakamoto', 'bitcoin', 'documentary', 'history']
|
||||
)
|
||||
claim4 = await self.stream_create(
|
||||
"satoshi-conspiracies", channel_id=chan3_id, title="Satoshi Nakamoto Conspiracies",
|
||||
description="Documentary detailing various conspiracies surrounding Satoshi Nakamoto.",
|
||||
tags=['conspiracies', 'bitcoin', 'satoshi nakamoto']
|
||||
)
|
||||
|
||||
await self.assertFindsClaims([], text='cheese')
|
||||
await self.assertFindsClaims([claim2], text='autobiography')
|
||||
await self.assertFindsClaims([claim3], text='history')
|
||||
await self.assertFindsClaims([claim4], text='conspiracy')
|
||||
await self.assertFindsClaims([], text='conspiracy AND history')
|
||||
await self.assertFindsClaims([claim4, claim3], text='conspiracy OR history')
|
||||
await self.assertFindsClaims([claim1, claim4, claim2, claim3], text='documentary')
|
||||
await self.assertFindsClaims([claim4, claim1, claim2, claim3], text='satoshi')
|
||||
|
||||
claim2 = await self.stream_update(
|
||||
self.get_claim_id(claim2), clear_tags=True, tags=['cloud'],
|
||||
title="Satoshi Nakamoto Nography",
|
||||
description="I am Satoshi Nakamoto and this is my nography.",
|
||||
)
|
||||
await self.assertFindsClaims([], text='autobiography')
|
||||
await self.assertFindsClaims([claim2], text='cloud')
|
||||
|
||||
await self.stream_abandon(self.get_claim_id(claim2))
|
||||
await self.assertFindsClaims([], text='cloud')
|
||||
|
||||
|
||||
class ChannelCommands(CommandTestCase):
|
||||
|
||||
@@ -664,12 +710,71 @@ class StreamCommands(ClaimTestCase):
|
||||
)
|
||||
|
||||
async def test_preview_works_with_signed_streams(self):
|
||||
await self.out(self.channel_create('@spam', '1.0'))
|
||||
signed = await self.out(self.stream_create('bar', '1.0', channel_name='@spam', preview=True, confirm=False))
|
||||
await self.channel_create('@spam', '1.0')
|
||||
signed = await self.stream_create('bar', '1.0', channel_name='@spam', preview=True, confirm=False)
|
||||
self.assertTrue(signed['outputs'][0]['is_channel_signature_valid'])
|
||||
|
||||
async def test_repost(self):
|
||||
await self.channel_create('@goodies', '1.0')
|
||||
tx = await self.stream_create('newstuff', '1.1', channel_name='@goodies')
|
||||
claim_id = self.get_claim_id(tx)
|
||||
|
||||
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 0)
|
||||
|
||||
tx = await self.stream_repost(claim_id, 'newstuff-again', '1.1')
|
||||
repost_id = self.get_claim_id(tx)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 1)
|
||||
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 1)
|
||||
self.assertEqual((await self.claim_search(reposted_claim_id=claim_id))[0]['claim_id'], repost_id)
|
||||
|
||||
await self.channel_create('@reposting-goodies', '1.0')
|
||||
await self.stream_repost(claim_id, 'repost-on-channel', '1.1', channel_name='@reposting-goodies')
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_list(claim_type='repost'), 2)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_claim_search(reposted_claim_id=claim_id), 2)
|
||||
self.assertEqual((await self.claim_search(name='newstuff'))[0]['meta']['reposted'], 2)
|
||||
|
||||
search_results = await self.claim_search(reposted='>=2')
|
||||
self.assertEqual(len(search_results), 1)
|
||||
self.assertEqual(search_results[0]['name'], 'newstuff')
|
||||
|
||||
search_results = await self.claim_search(name='repost-on-channel')
|
||||
self.assertEqual(len(search_results), 1)
|
||||
search = search_results[0]
|
||||
self.assertEqual(search['name'], 'repost-on-channel')
|
||||
self.assertEqual(search['signing_channel']['name'], '@reposting-goodies')
|
||||
self.assertEqual(search['reposted_claim']['name'], 'newstuff')
|
||||
self.assertEqual(search['reposted_claim']['meta']['reposted'], 2)
|
||||
self.assertEqual(search['reposted_claim']['signing_channel']['name'], '@goodies')
|
||||
|
||||
resolved = await self.resolve(['@reposting-goodies/repost-on-channel', 'newstuff-again'])
|
||||
self.assertEqual(resolved['@reposting-goodies/repost-on-channel'], search)
|
||||
self.assertEqual(resolved['newstuff-again']['reposted_claim']['name'], 'newstuff')
|
||||
|
||||
async def test_filtering_channels_for_removing_content(self):
|
||||
await self.channel_create('@badstuff', '1.0')
|
||||
await self.stream_create('not_bad', '1.1', channel_name='@badstuff')
|
||||
tx = await self.stream_create('too_bad', '1.1', channel_name='@badstuff')
|
||||
claim_id = self.get_claim_id(tx)
|
||||
await self.channel_create('@reposts', '1.0')
|
||||
await self.stream_repost(claim_id, 'normal_repost', '1.2', channel_name='@reposts')
|
||||
filtering1 = await self.channel_create('@filtering1', '1.0')
|
||||
filtering1 = self.get_claim_id(filtering1)
|
||||
await self.stream_repost(claim_id, 'filter1', '1.1', channel_name='@filtering1')
|
||||
await self.conductor.spv_node.stop()
|
||||
await self.conductor.spv_node.start(
|
||||
self.conductor.blockchain_node, extraconf={'FILTERING_CHANNELS_IDS': filtering1}
|
||||
)
|
||||
await self.ledger.stop()
|
||||
await self.ledger.start()
|
||||
filtered_claim_search = await self.claim_search(name='too_bad')
|
||||
self.assertEqual(filtered_claim_search, [])
|
||||
filtered_claim_search = await self.claim_search(name='not_bad')
|
||||
self.assertEqual(len(filtered_claim_search), 1)
|
||||
filtered_claim_search = await self.claim_search(name='normal_repost')
|
||||
self.assertEqual(len(filtered_claim_search), 1)
|
||||
|
||||
async def test_publish_updates_file_list(self):
|
||||
tx = await self.out(self.stream_create(title='created'))
|
||||
tx = await self.stream_create(title='created')
|
||||
txo = tx['outputs'][0]
|
||||
claim_id, expected = txo['claim_id'], txo['value']
|
||||
files = await self.file_list()
|
||||
@@ -678,7 +783,7 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual(expected, files[0]['metadata'])
|
||||
|
||||
# update with metadata-only changes
|
||||
tx = await self.out(self.stream_update(claim_id, title='update 1'))
|
||||
tx = await self.stream_update(claim_id, title='update 1')
|
||||
files = await self.file_list()
|
||||
expected['title'] = 'update 1'
|
||||
self.assertEqual(1, len(files))
|
||||
@@ -686,7 +791,7 @@ class StreamCommands(ClaimTestCase):
|
||||
self.assertEqual(expected, files[0]['metadata'])
|
||||
|
||||
# update with new data
|
||||
tx = await self.out(self.stream_update(claim_id, title='update 2', data=b'updated data'))
|
||||
tx = await self.stream_update(claim_id, title='update 2', data=b'updated data')
|
||||
expected = tx['outputs'][0]['value']
|
||||
files = await self.file_list()
|
||||
self.assertEqual(1, len(files))
|
||||
@@ -1211,3 +1316,73 @@ class SupportCommands(CommandTestCase):
|
||||
self.assertFalse(txs2[0]['support_info'][0]['is_tip'])
|
||||
self.assertEqual(txs2[0]['value'], '0.0')
|
||||
self.assertEqual(txs2[0]['fee'], '-0.0001415')
|
||||
|
||||
|
||||
class CollectionCommands(CommandTestCase):
|
||||
|
||||
async def test_collections(self):
|
||||
claim_ids = [
|
||||
self.get_claim_id(tx) for tx in [
|
||||
await self.stream_create('stream-one'),
|
||||
await self.stream_create('stream-two')
|
||||
]
|
||||
]
|
||||
claim_ids.append(claim_ids[0])
|
||||
claim_ids.append('beef')
|
||||
tx = await self.collection_create('radjingles', claims=claim_ids, title="boring title")
|
||||
claim_id = self.get_claim_id(tx)
|
||||
collections = await self.out(self.daemon.jsonrpc_collection_list())
|
||||
self.assertEqual(collections['items'][0]['value']['title'], 'boring title')
|
||||
self.assertEqual(collections['items'][0]['value']['claims'], claim_ids)
|
||||
self.assertEqual(collections['items'][0]['value_type'], 'collection')
|
||||
|
||||
self.assertItemCount(collections, 1)
|
||||
await self.assertBalance(self.account, '6.939679')
|
||||
|
||||
with self.assertRaisesRegex(Exception, "You already have a collection under the name 'radjingles'."):
|
||||
await self.collection_create('radjingles', claims=claim_ids)
|
||||
|
||||
self.assertItemCount(await self.daemon.jsonrpc_collection_list(), 1)
|
||||
await self.assertBalance(self.account, '6.939679')
|
||||
|
||||
collections = await self.out(self.daemon.jsonrpc_collection_list())
|
||||
self.assertEqual(collections['items'][0]['value']['title'], 'boring title')
|
||||
await self.collection_update(claim_id, title='fancy title')
|
||||
collections = await self.out(self.daemon.jsonrpc_collection_list())
|
||||
self.assertEqual(collections['items'][0]['value']['title'], 'fancy title')
|
||||
self.assertEqual(collections['items'][0]['value']['claims'], claim_ids)
|
||||
self.assertNotIn('claims', collections['items'][0])
|
||||
|
||||
tx = await self.collection_create('radjingles', claims=claim_ids, allow_duplicate_name=True)
|
||||
claim_id2 = self.get_claim_id(tx)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_collection_list(), 2)
|
||||
|
||||
await self.collection_update(claim_id, clear_claims=True, claims=claim_ids[:2])
|
||||
collections = await self.out(self.daemon.jsonrpc_collection_list())
|
||||
self.assertEquals(len(collections['items']), 2)
|
||||
|
||||
await self.collection_abandon(claim_id)
|
||||
self.assertItemCount(await self.daemon.jsonrpc_collection_list(), 1)
|
||||
|
||||
collections = await self.out(self.daemon.jsonrpc_collection_list(resolve_claims=2))
|
||||
self.assertEquals(len(collections['items'][0]['claims']), 2)
|
||||
|
||||
collections = await self.out(self.daemon.jsonrpc_collection_list(resolve_claims=10))
|
||||
self.assertEqual(len(collections['items'][0]['claims']), 4)
|
||||
self.assertEqual(collections['items'][0]['claims'][0]['name'], 'stream-one')
|
||||
self.assertEqual(collections['items'][0]['claims'][1]['name'], 'stream-two')
|
||||
self.assertEqual(collections['items'][0]['claims'][2]['name'], 'stream-one')
|
||||
self.assertIsNone(collections['items'][0]['claims'][3])
|
||||
|
||||
claims = await self.out(self.daemon.jsonrpc_claim_list())
|
||||
self.assertEqual(claims['items'][0]['name'], 'radjingles')
|
||||
self.assertEqual(claims['items'][1]['name'], 'stream-two')
|
||||
self.assertEqual(claims['items'][2]['name'], 'stream-one')
|
||||
|
||||
claims = await self.out(self.daemon.jsonrpc_collection_resolve(claim_id2))
|
||||
self.assertEqual(claims['items'][0]['name'], 'stream-one')
|
||||
self.assertEqual(claims['items'][1]['name'], 'stream-two')
|
||||
self.assertEqual(claims['items'][2]['name'], 'stream-one')
|
||||
|
||||
claims = await self.out(self.daemon.jsonrpc_collection_resolve(claim_id2, page=10))
|
||||
self.assertEqual(claims['items'], [])
|
||||
@@ -5,7 +5,7 @@ from torba.testcase import AsyncioTestCase
|
||||
from lbry.conf import Config
|
||||
from lbry.extras import cli
|
||||
from lbry.extras.daemon.Components import (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
)
|
||||
@@ -20,7 +20,7 @@ class CLIIntegrationTest(AsyncioTestCase):
|
||||
conf.share_usage_data = False
|
||||
conf.api = 'localhost:5299'
|
||||
conf.components_to_skip = (
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
)
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.conf import Config
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht import peer as dht_peer
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
@@ -10,30 +13,40 @@ from torba.testcase import AsyncioTestCase
|
||||
class DHTIntegrationTest(AsyncioTestCase):
|
||||
|
||||
async def asyncSetUp(self):
|
||||
dht_peer.ALLOW_LOCALHOST = True
|
||||
self.addCleanup(setattr, dht_peer, 'ALLOW_LOCALHOST', False)
|
||||
import logging
|
||||
logging.getLogger('asyncio').setLevel(logging.ERROR)
|
||||
logging.getLogger('lbry.dht').setLevel(logging.WARN)
|
||||
self.nodes = []
|
||||
self.known_node_addresses = []
|
||||
|
||||
async def setup_network(self, size: int, start_port=40000, seed_nodes=1):
|
||||
async def create_node(self, node_id, port, external_ip='127.0.0.1'):
|
||||
storage = SQLiteStorage(Config(), ":memory:", self.loop, self.loop.time)
|
||||
await storage.open()
|
||||
node = Node(self.loop, PeerManager(self.loop), node_id=node_id,
|
||||
udp_port=port, internal_udp_port=port,
|
||||
peer_port=3333, external_ip=external_ip,
|
||||
storage=storage)
|
||||
self.addCleanup(node.stop)
|
||||
node.protocol.rpc_timeout = .5
|
||||
node.protocol.ping_queue._default_delay = .5
|
||||
return node
|
||||
|
||||
async def setup_network(self, size: int, start_port=40000, seed_nodes=1, external_ip='127.0.0.1'):
|
||||
for i in range(size):
|
||||
node_port = start_port + i
|
||||
node = Node(self.loop, PeerManager(self.loop), node_id=constants.generate_id(i),
|
||||
udp_port=node_port, internal_udp_port=node_port,
|
||||
peer_port=3333, external_ip='127.0.0.1')
|
||||
node_id = constants.generate_id(i)
|
||||
node = await self.create_node(node_id, node_port)
|
||||
self.nodes.append(node)
|
||||
self.known_node_addresses.append(('127.0.0.1', node_port))
|
||||
await node.start_listening('127.0.0.1')
|
||||
self.addCleanup(node.stop)
|
||||
self.known_node_addresses.append((external_ip, node_port))
|
||||
|
||||
for node in self.nodes:
|
||||
node.protocol.rpc_timeout = .5
|
||||
node.protocol.ping_queue._default_delay = .5
|
||||
node.start('127.0.0.1', self.known_node_addresses[:seed_nodes])
|
||||
await asyncio.gather(*[node.joined.wait() for node in self.nodes])
|
||||
node.start(external_ip, self.known_node_addresses[:seed_nodes])
|
||||
|
||||
async def test_replace_bad_nodes(self):
|
||||
await self.setup_network(20)
|
||||
await asyncio.gather(*[node.joined.wait() for node in self.nodes])
|
||||
self.assertEqual(len(self.nodes), 20)
|
||||
node = self.nodes[0]
|
||||
bad_peers = []
|
||||
@@ -54,6 +67,7 @@ class DHTIntegrationTest(AsyncioTestCase):
|
||||
|
||||
async def test_re_join(self):
|
||||
await self.setup_network(20, seed_nodes=10)
|
||||
await asyncio.gather(*[node.joined.wait() for node in self.nodes])
|
||||
node = self.nodes[-1]
|
||||
self.assertTrue(node.joined.is_set())
|
||||
self.assertTrue(node.protocol.routing_table.get_peers())
|
||||
@@ -81,6 +95,7 @@ class DHTIntegrationTest(AsyncioTestCase):
|
||||
|
||||
async def test_get_token_on_announce(self):
|
||||
await self.setup_network(2, seed_nodes=2)
|
||||
await asyncio.gather(*[node.joined.wait() for node in self.nodes])
|
||||
node1, node2 = self.nodes
|
||||
node1.protocol.peer_manager.clear_token(node2.protocol.node_id)
|
||||
blob_hash = hexlify(constants.generate_id(1337)).decode()
|
||||
@@ -98,6 +113,7 @@ class DHTIntegrationTest(AsyncioTestCase):
|
||||
# imagine that you only got bad peers and refresh will happen in one hour
|
||||
# instead of failing for one hour we should be able to recover by scheduling pings to bad peers we find
|
||||
await self.setup_network(2, seed_nodes=2)
|
||||
await asyncio.gather(*[node.joined.wait() for node in self.nodes])
|
||||
node1, node2 = self.nodes
|
||||
node2.stop()
|
||||
# forcefully make it a bad peer but don't remove it from routing table
|
||||
@@ -113,3 +129,37 @@ class DHTIntegrationTest(AsyncioTestCase):
|
||||
await node1.peer_search(node2.protocol.node_id)
|
||||
await asyncio.sleep(.3) # let pending events settle
|
||||
self.assertFalse(node1.protocol.routing_table.get_peers())
|
||||
|
||||
async def test_peer_persistance(self):
|
||||
num_nodes = 6
|
||||
start_port = 40000
|
||||
num_seeds = 2
|
||||
external_ip = '127.0.0.1'
|
||||
|
||||
# Start a node
|
||||
await self.setup_network(num_nodes, start_port=start_port, seed_nodes=num_seeds)
|
||||
await asyncio.gather(*[node.joined.wait() for node in self.nodes])
|
||||
|
||||
node1 = self.nodes[-1]
|
||||
peer_args = [(n.protocol.node_id, n.protocol.external_ip, n.protocol.udp_port, n.protocol.peer_port) for n in
|
||||
self.nodes[:num_seeds]]
|
||||
peers = [make_kademlia_peer(*args) for args in peer_args]
|
||||
|
||||
# node1 is bootstrapped from the fixed seeds
|
||||
self.assertCountEqual(peers, node1.protocol.routing_table.get_peers())
|
||||
|
||||
# Refresh and assert that the peers were persisted
|
||||
await node1.refresh_node(True)
|
||||
self.assertEqual(len(peer_args), len(await node1._storage.get_persisted_kademlia_peers()))
|
||||
node1.stop()
|
||||
|
||||
# Start a fresh node with the same node_id and storage, but no known peers
|
||||
node2 = await self.create_node(constants.generate_id(num_nodes-1), start_port+num_nodes-1)
|
||||
node2._storage = node1._storage
|
||||
node2.start(external_ip, [])
|
||||
await node2.joined.wait()
|
||||
|
||||
# The peers are restored
|
||||
self.assertEqual(num_seeds, len(node2.protocol.routing_table.get_peers()))
|
||||
for bucket1, bucket2 in zip(node1.protocol.routing_table.buckets, node2.protocol.routing_table.buckets):
|
||||
self.assertEqual((bucket1.range_min, bucket1.range_max), (bucket2.range_min, bucket2.range_max))
|
||||
@@ -205,12 +205,12 @@ class FileCommands(CommandTestCase):
|
||||
await self.server.blob_manager.delete_blobs(all_except_sd)
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
|
||||
self.assertIn('error', resp)
|
||||
self.assertEqual('Failed to download data blobs for sd hash %s within timeout' % sd_hash, resp['error'])
|
||||
self.assertEqual('Failed to download data blobs for sd hash %s within timeout.' % sd_hash, resp['error'])
|
||||
self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo'), "data timeout didn't create a file")
|
||||
await self.server.blob_manager.delete_blobs([sd_hash])
|
||||
resp = await self.daemon.jsonrpc_get('lbry://foo', timeout=2, save_file=True)
|
||||
self.assertIn('error', resp)
|
||||
self.assertEqual('Failed to download sd blob %s within timeout' % sd_hash, resp['error'])
|
||||
self.assertEqual('Failed to download sd blob %s within timeout.' % sd_hash, resp['error'])
|
||||
|
||||
async def wait_files_to_complete(self):
|
||||
while await self.file_list(status='running'):
|
||||
@@ -294,26 +294,6 @@ class FileCommands(CommandTestCase):
|
||||
self.assertEqual(file_info['blobs_completed'], file_info['blobs_in_stream'])
|
||||
self.assertEqual('finished', file_info['status'])
|
||||
|
||||
async def test_unban_recovers_stream(self):
|
||||
BlobDownloader.BAN_FACTOR = .5 # fixme: temporary field, will move to connection manager or a conf
|
||||
tx = await self.stream_create('foo', '0.01', data=bytes([0] * (1 << 23)))
|
||||
sd_hash = tx['outputs'][0]['value']['source']['sd_hash']
|
||||
missing_blob_hash = (await self.daemon.jsonrpc_blob_list(sd_hash=sd_hash))['items'][-2]
|
||||
await self.daemon.jsonrpc_file_delete(claim_name='foo')
|
||||
# backup blob
|
||||
missing_blob = self.server_blob_manager.get_blob(missing_blob_hash)
|
||||
os.rename(missing_blob.file_path, missing_blob.file_path + '__')
|
||||
self.server_blob_manager.delete_blob(missing_blob_hash)
|
||||
await self.daemon.jsonrpc_get('lbry://foo')
|
||||
with self.assertRaises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
|
||||
# restore blob
|
||||
os.rename(missing_blob.file_path + '__', missing_blob.file_path)
|
||||
self.server_blob_manager.blobs.clear()
|
||||
missing_blob = self.server_blob_manager.get_blob(missing_blob_hash)
|
||||
self.server_blob_manager.blob_completed(missing_blob)
|
||||
await asyncio.wait_for(self.wait_files_to_complete(), timeout=1)
|
||||
|
||||
async def test_paid_download(self):
|
||||
target_address = await self.blockchain.get_raw_change_address()
|
||||
|
||||
|
||||
@@ -10,3 +10,16 @@ class AddressManagement(CommandTestCase):
|
||||
single = await self.out(self.daemon.jsonrpc_address_list(addresses['items'][11]['address']))
|
||||
self.assertItemCount(single, 1)
|
||||
self.assertEqual(single['items'][0], addresses['items'][11])
|
||||
|
||||
class SettingsManagement(CommandTestCase):
|
||||
|
||||
async def test_settings(self):
|
||||
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('127.0.0.1', 50001))
|
||||
|
||||
setting = self.daemon.jsonrpc_settings_set('lbryum_servers', ['server:50001'])
|
||||
self.assertEqual(setting['lbryum_servers'][0], ('server', 50001))
|
||||
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('server', 50001))
|
||||
|
||||
setting = self.daemon.jsonrpc_settings_clear('lbryum_servers')
|
||||
self.assertEqual(setting['lbryum_servers'][0], ('spv1.lbry.com', 50001))
|
||||
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv1.lbry.com', 50001))
|
||||
@@ -1,10 +1,7 @@
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import lbry.wallet
|
||||
from lbry import __version__ as sdk_version
|
||||
from lbry.testcase import CommandTestCase
|
||||
from lbry.extras.daemon.Components import HeadersComponent
|
||||
from torba.client.basenetwork import ClientSession
|
||||
from torba.testcase import IntegrationTestCase
|
||||
|
||||
@@ -38,6 +35,13 @@ class TestSessions(IntegrationTestCase):
|
||||
info = await self.ledger.network.get_server_features()
|
||||
self.assertEqual(sdk_version, info['server_version'])
|
||||
|
||||
async def test_client_errors(self):
|
||||
# Goal is ensuring thsoe are raised and not trapped accidentally
|
||||
with self.assertRaisesRegex(Exception, 'not a valid address'):
|
||||
await self.ledger.network.get_history('of the world')
|
||||
with self.assertRaisesRegex(Exception, 'rejected by network rules.*TX decode failed'):
|
||||
await self.ledger.network.broadcast('13370042004200')
|
||||
|
||||
|
||||
class TestSegwitServer(IntegrationTestCase):
|
||||
LEDGER = lbry.wallet
|
||||
@@ -45,22 +49,3 @@ class TestSegwitServer(IntegrationTestCase):
|
||||
|
||||
async def test_at_least_it_starts(self):
|
||||
await asyncio.wait_for(self.ledger.network.get_headers(0, 1), 1.0)
|
||||
|
||||
|
||||
class TestHeadersComponent(CommandTestCase):
|
||||
|
||||
LEDGER = lbry.wallet
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
self.component_manager = self.daemon.component_manager
|
||||
self.component_manager.conf.blockchain_name = 'lbrycrd_main'
|
||||
self.headers_component = HeadersComponent(self.component_manager)
|
||||
|
||||
async def test_cant_reach_host(self):
|
||||
HeadersComponent.HEADERS_URL = 'notthere/'
|
||||
os.unlink(self.headers_component.headers.path)
|
||||
# test is that this doesn't raise
|
||||
await self.headers_component.start()
|
||||
self.assertTrue(self.component_manager.get_components_status()['blockchain_headers'])
|
||||
self.assertEqual(await self.headers_component.get_status(), {})
|
||||
@@ -44,7 +44,7 @@ class BlobExchangeTestBase(AsyncioTestCase):
|
||||
self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite"))
|
||||
self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config)
|
||||
self.client_peer_manager = PeerManager(self.loop)
|
||||
self.server_from_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333)
|
||||
self.server_from_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333, allow_localhost=True)
|
||||
|
||||
await self.client_storage.open()
|
||||
await self.server_storage.open()
|
||||
@@ -103,7 +103,7 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
second_client_blob_manager = BlobManager(
|
||||
self.loop, second_client_dir, second_client_storage, second_client_conf
|
||||
)
|
||||
server_from_second_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333)
|
||||
server_from_second_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333, allow_localhost=True)
|
||||
|
||||
await second_client_storage.open()
|
||||
await second_client_blob_manager.setup()
|
||||
@@ -194,7 +194,7 @@ class TestBlobExchange(BlobExchangeTestBase):
|
||||
second_client_blob_manager = BlobManager(
|
||||
self.loop, second_client_dir, second_client_storage, second_client_conf
|
||||
)
|
||||
server_from_second_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333)
|
||||
server_from_second_client = make_kademlia_peer(b'1' * 48, "127.0.0.1", tcp_port=33333, allow_localhost=True)
|
||||
|
||||
await second_client_storage.open()
|
||||
await second_client_blob_manager.setup()
|
||||
|
||||
@@ -6,7 +6,6 @@ from lbry.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbry.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT
|
||||
from lbry.extras.daemon.Components import HASH_ANNOUNCER_COMPONENT, UPNP_COMPONENT
|
||||
from lbry.extras.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
from lbry.extras.daemon.Components import HEADERS_COMPONENT
|
||||
from lbry.extras.daemon import Components
|
||||
|
||||
|
||||
@@ -15,7 +14,6 @@ class TestComponentManager(AsyncioTestCase):
|
||||
def setUp(self):
|
||||
self.default_components_sort = [
|
||||
[
|
||||
Components.HeadersComponent,
|
||||
Components.DatabaseComponent,
|
||||
Components.ExchangeRateManagerComponent,
|
||||
Components.UPnPComponent
|
||||
@@ -152,7 +150,7 @@ class TestComponentManagerProperStart(AdvanceTimeTestCase):
|
||||
skip_components=[
|
||||
DATABASE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT,
|
||||
HEADERS_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT],
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT],
|
||||
wallet=FakeDelayedWallet,
|
||||
stream_manager=FakeDelayedStreamManager,
|
||||
blob_manager=FakeDelayedBlobManager
|
||||
|
||||
@@ -3,6 +3,7 @@ import tempfile
|
||||
import unittest
|
||||
import asyncio
|
||||
import logging
|
||||
import hashlib
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
@@ -10,6 +11,7 @@ from lbry.blob.blob_info import BlobInfo
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from tests.test_utils import random_lbry_hash
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
|
||||
log = logging.getLogger()
|
||||
|
||||
@@ -247,3 +249,13 @@ class ContentClaimStorageTests(StorageTest):
|
||||
current_claim_info = await self.storage.get_content_claim(stream_hash)
|
||||
# this should still be the previous update
|
||||
self.assertDictEqual(current_claim_info, update_info)
|
||||
|
||||
|
||||
class UpdatePeersTest(StorageTest):
|
||||
async def test_update_get_peers(self):
|
||||
node_id = hashlib.sha384("1234".encode()).digest()
|
||||
args = (node_id, '73.186.148.72', 4444, None)
|
||||
fake_peer = make_kademlia_peer(*args)
|
||||
await self.storage.save_kademlia_peers([fake_peer])
|
||||
peers = await self.storage.get_persisted_kademlia_peers()
|
||||
self.assertTupleEqual(args, peers[0])
|
||||
@@ -7,7 +7,7 @@ from lbry.dht import constants
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
|
||||
def address_generator(address=(10, 42, 42, 1)):
|
||||
def address_generator(address=(1, 2, 3, 4)):
|
||||
def increment(addr):
|
||||
value = struct.unpack("I", "".join([chr(x) for x in list(addr)[::-1]]).encode())[0] + 1
|
||||
new_addr = []
|
||||
|
||||
@@ -119,7 +119,7 @@ class TestBlobAnnouncer(AsyncioTestCase):
|
||||
async def test_popular_blob(self):
|
||||
peer_count = 150
|
||||
addresses = [
|
||||
(constants.generate_id(i + 1), socket.inet_ntoa(int(i + 1).to_bytes(length=4, byteorder='big')))
|
||||
(constants.generate_id(i + 1), socket.inet_ntoa(int(i + 0x01000001).to_bytes(length=4, byteorder='big')))
|
||||
for i in range(peer_count)
|
||||
]
|
||||
blob_hash = b'1' * 48
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import asyncio
|
||||
import unittest
|
||||
import typing
|
||||
from torba.testcase import AsyncioTestCase
|
||||
from tests import dht_mocks
|
||||
from lbry.conf import Config
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import PeerManager, make_kademlia_peer
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
|
||||
class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
@@ -84,3 +87,75 @@ class TestNodePingQueueDiscover(AsyncioTestCase):
|
||||
# teardown
|
||||
for n in nodes.values():
|
||||
n.stop()
|
||||
|
||||
|
||||
class TestTemporarilyLosingConnection(AsyncioTestCase):
|
||||
@unittest.SkipTest
|
||||
async def test_losing_connection(self):
|
||||
async def wait_for(check_ok, insist, timeout=20):
|
||||
start = loop.time()
|
||||
while loop.time() - start < timeout:
|
||||
if check_ok():
|
||||
break
|
||||
await asyncio.sleep(0)
|
||||
else:
|
||||
insist()
|
||||
|
||||
loop = self.loop
|
||||
loop.set_debug(False)
|
||||
|
||||
peer_addresses = [
|
||||
('1.2.3.4', 40000+i) for i in range(10)
|
||||
]
|
||||
node_ids = [constants.generate_id(i) for i in range(10)]
|
||||
|
||||
nodes = [
|
||||
Node(
|
||||
loop, PeerManager(loop), node_id, udp_port, udp_port, 3333, address,
|
||||
storage=SQLiteStorage(Config(), ":memory:", self.loop, self.loop.time)
|
||||
)
|
||||
for node_id, (address, udp_port) in zip(node_ids, peer_addresses)
|
||||
]
|
||||
dht_network = {peer_addresses[i]: node.protocol for i, node in enumerate(nodes)}
|
||||
num_seeds = 3
|
||||
|
||||
with dht_mocks.mock_network_loop(loop, dht_network):
|
||||
for i, n in enumerate(nodes):
|
||||
await n._storage.open()
|
||||
self.addCleanup(n.stop)
|
||||
n.start(peer_addresses[i][0], peer_addresses[:num_seeds])
|
||||
await asyncio.gather(*[n.joined.wait() for n in nodes])
|
||||
|
||||
node = nodes[-1]
|
||||
advance = dht_mocks.get_time_accelerator(loop, loop.time())
|
||||
await advance(500)
|
||||
|
||||
# Join the network, assert that at least the known peers are in RT
|
||||
self.assertTrue(node.joined.is_set())
|
||||
self.assertTrue(len(node.protocol.routing_table.get_peers()) >= num_seeds)
|
||||
|
||||
# Refresh, so that the peers are persisted
|
||||
self.assertFalse(len(await node._storage.get_persisted_kademlia_peers()) > num_seeds)
|
||||
await advance(4000)
|
||||
self.assertTrue(len(await node._storage.get_persisted_kademlia_peers()) > num_seeds)
|
||||
|
||||
# We lost internet connection - all the peers stop responding
|
||||
dht_network.pop((node.protocol.external_ip, node.protocol.udp_port))
|
||||
|
||||
# The peers are cleared on refresh from RT and storage
|
||||
await advance(4000)
|
||||
self.assertListEqual([], await node._storage.get_persisted_kademlia_peers())
|
||||
await wait_for(
|
||||
lambda: len(node.protocol.routing_table.get_peers()) == 0,
|
||||
lambda: self.assertListEqual(node.protocol.routing_table.get_peers(), [])
|
||||
)
|
||||
|
||||
# Reconnect
|
||||
dht_network[(node.protocol.external_ip, node.protocol.udp_port)] = node.protocol
|
||||
|
||||
# Check that node reconnects at least to them
|
||||
await advance(1000)
|
||||
await wait_for(
|
||||
lambda: len(node.protocol.routing_table.get_peers()) >= num_seeds,
|
||||
lambda: self.assertTrue(len(node.protocol.routing_table.get_peers()) >= num_seeds)
|
||||
)
|
||||
@@ -10,8 +10,8 @@ class PeerTest(AsyncioTestCase):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.peer_manager = PeerManager(self.loop)
|
||||
self.node_ids = [generate_id(), generate_id(), generate_id()]
|
||||
self.first_contact = make_kademlia_peer(self.node_ids[1], '127.0.0.1', udp_port=1000)
|
||||
self.second_contact = make_kademlia_peer(self.node_ids[0], '192.168.0.1', udp_port=1000)
|
||||
self.first_contact = make_kademlia_peer(self.node_ids[1], '1.0.0.1', udp_port=1000)
|
||||
self.second_contact = make_kademlia_peer(self.node_ids[0], '1.0.0.2', udp_port=1000)
|
||||
|
||||
def test_peer_is_good_unknown_peer(self):
|
||||
# Scenario: peer replied, but caller doesn't know the node_id.
|
||||
@@ -24,21 +24,42 @@ class PeerTest(AsyncioTestCase):
|
||||
self.assertIsNone(self.peer_manager.peer_is_good(peer))
|
||||
|
||||
def test_make_contact_error_cases(self):
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.168.1.20', 100000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.168.1.20.1', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '1.2.3.4', 100000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '1.2.3.4.5', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], 'this is not an ip', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.168.1.20', -1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, b'not valid node id', '192.168.1.20', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '1.2.3.4', -1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '1.2.3.4', 0)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '1.2.3.4', 70000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, b'not valid node id', '1.2.3.4', 1000)
|
||||
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '0.0.0.0', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '10.0.0.1', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '100.64.0.1', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '127.0.0.1', 1000)
|
||||
self.assertIsNotNone(make_kademlia_peer(self.node_ids[1], '127.0.0.1', 1000, allow_localhost=True))
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.168.0.1', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '172.16.0.1', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '169.254.1.1', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.0.0.2', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.0.2.2', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '192.88.99.2', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '198.18.1.1', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '198.51.100.2', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '198.51.100.2', 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '203.0.113.4', 1000)
|
||||
for i in range(32):
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], f"{224 + i}.0.0.0", 1000)
|
||||
self.assertRaises(ValueError, make_kademlia_peer, self.node_ids[1], '255.255.255.255', 1000)
|
||||
|
||||
def test_boolean(self):
|
||||
self.assertNotEqual(self.first_contact, self.second_contact)
|
||||
self.assertEqual(
|
||||
self.second_contact, make_kademlia_peer(self.node_ids[0], '192.168.0.1', udp_port=1000)
|
||||
self.second_contact, make_kademlia_peer(self.node_ids[0], '1.0.0.2', udp_port=1000)
|
||||
)
|
||||
|
||||
def test_compact_ip(self):
|
||||
self.assertEqual(self.first_contact.compact_ip(), b'\x7f\x00\x00\x01')
|
||||
self.assertEqual(self.second_contact.compact_ip(), b'\xc0\xa8\x00\x01')
|
||||
self.assertEqual(b'\x01\x00\x00\x01', self.first_contact.compact_ip())
|
||||
self.assertEqual(b'\x01\x00\x00\x02', self.second_contact.compact_ip())
|
||||
|
||||
|
||||
@unittest.SkipTest
|
||||
|
||||
@@ -9,7 +9,6 @@ from lbry.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, WAL
|
||||
from lbry.extras.daemon.Components import HASH_ANNOUNCER_COMPONENT
|
||||
from lbry.extras.daemon.Components import UPNP_COMPONENT, BLOB_COMPONENT
|
||||
from lbry.extras.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
from lbry.extras.daemon.Components import HEADERS_COMPONENT
|
||||
from lbry.extras.daemon.Daemon import Daemon as LBRYDaemon
|
||||
from lbry.wallet import LbryWalletManager
|
||||
from torba.client.wallet import Wallet
|
||||
@@ -31,7 +30,7 @@ def get_test_daemon(conf: Config, with_fee=False):
|
||||
conf, skip_components=[
|
||||
DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, UPNP_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT,
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
|
||||
RATE_LIMITER_COMPONENT],
|
||||
file_manager=FakeFileManager
|
||||
)
|
||||
|
||||
@@ -2,7 +2,7 @@ import unittest
|
||||
from decimal import Decimal
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.extras.daemon import exchange_rate_manager
|
||||
from lbry.error import InvalidExchangeRateResponse
|
||||
from lbry.error import InvalidExchangeRateResponseError
|
||||
from tests import test_utils
|
||||
|
||||
|
||||
@@ -101,11 +101,11 @@ class LBRYioFeedTest(unittest.TestCase):
|
||||
self.assertEqual(expected, out)
|
||||
|
||||
response = '{}'
|
||||
with self.assertRaises(InvalidExchangeRateResponse):
|
||||
with self.assertRaises(InvalidExchangeRateResponseError):
|
||||
feed._handle_response(response)
|
||||
|
||||
response = '{"success":true,"result":[]}'
|
||||
with self.assertRaises(InvalidExchangeRateResponse):
|
||||
with self.assertRaises(InvalidExchangeRateResponseError):
|
||||
feed._handle_response(response)
|
||||
|
||||
|
||||
@@ -121,11 +121,11 @@ class TestExchangeRateFeeds(unittest.TestCase):
|
||||
self.assertEqual(expected, out)
|
||||
|
||||
response = '{}'
|
||||
with self.assertRaises(InvalidExchangeRateResponse):
|
||||
with self.assertRaises(InvalidExchangeRateResponseError):
|
||||
out = feed._handle_response(response)
|
||||
|
||||
response = '{"success":true,"result":[]}'
|
||||
with self.assertRaises(InvalidExchangeRateResponse):
|
||||
with self.assertRaises(InvalidExchangeRateResponseError):
|
||||
out = feed._handle_response(response)
|
||||
|
||||
def test_handle_cryptonator_lbc_response(self):
|
||||
@@ -139,11 +139,11 @@ class TestExchangeRateFeeds(unittest.TestCase):
|
||||
self.assertEqual(expected, out)
|
||||
|
||||
response = '{}'
|
||||
with self.assertRaises(InvalidExchangeRateResponse):
|
||||
with self.assertRaises(InvalidExchangeRateResponseError):
|
||||
feed._handle_response(response)
|
||||
|
||||
response = '{"success":true,"ticker":{}}'
|
||||
with self.assertRaises(InvalidExchangeRateResponse):
|
||||
with self.assertRaises(InvalidExchangeRateResponseError):
|
||||
feed._handle_response(response)
|
||||
|
||||
def test_handle_cryptonator_btc_response(self):
|
||||
@@ -157,11 +157,11 @@ class TestExchangeRateFeeds(unittest.TestCase):
|
||||
self.assertEqual(expected, out)
|
||||
|
||||
response = '{}'
|
||||
with self.assertRaises(InvalidExchangeRateResponse):
|
||||
with self.assertRaises(InvalidExchangeRateResponseError):
|
||||
feed._handle_response(response)
|
||||
|
||||
response = '{"success":true,"ticker":{}}'
|
||||
with self.assertRaises(InvalidExchangeRateResponse):
|
||||
with self.assertRaises(InvalidExchangeRateResponseError):
|
||||
feed._handle_response(response)
|
||||
|
||||
def test_handle_bittrex_response(self):
|
||||
@@ -177,9 +177,9 @@ class TestExchangeRateFeeds(unittest.TestCase):
|
||||
self.assertEqual(expected, out)
|
||||
|
||||
response = '{}'
|
||||
with self.assertRaises(InvalidExchangeRateResponse):
|
||||
with self.assertRaises(InvalidExchangeRateResponseError):
|
||||
feed._handle_response(response)
|
||||
|
||||
response = '{"success":true,"result":[]}'
|
||||
with self.assertRaises(InvalidExchangeRateResponse):
|
||||
with self.assertRaises(InvalidExchangeRateResponseError):
|
||||
feed._handle_response(response)
|
||||
@@ -1,7 +1,7 @@
|
||||
from unittest import TestCase
|
||||
from decimal import Decimal
|
||||
|
||||
from lbry.schema.claim import Claim, Stream
|
||||
from lbry.schema.claim import Claim, Stream, Collection
|
||||
|
||||
|
||||
class TestClaimContainerAwareness(TestCase):
|
||||
@@ -118,6 +118,26 @@ class TestTags(TestCase):
|
||||
self.assertEqual(claim.channel.tags, ['anime'])
|
||||
|
||||
|
||||
class TestCollection(TestCase):
|
||||
|
||||
def test_collection(self):
|
||||
collection = Collection()
|
||||
|
||||
collection.update(claims=['abc123', 'def123'])
|
||||
self.assertListEqual(collection.claims.ids, ['abc123', 'def123'])
|
||||
|
||||
collection.update(claims=['abc123', 'bbb123'])
|
||||
self.assertListEqual(collection.claims.ids, ['abc123', 'def123', 'abc123', 'bbb123'])
|
||||
|
||||
collection.update(clear_claims=True, claims=['bbb987', 'bb'])
|
||||
self.assertListEqual(collection.claims.ids, ['bbb987', 'bb'])
|
||||
|
||||
self.assertEqual(collection.to_dict(), {'claims': ['bbb987', 'bb']})
|
||||
|
||||
collection.update(clear_claims=True)
|
||||
self.assertListEqual(collection.claims.ids, [])
|
||||
|
||||
|
||||
class TestLocations(TestCase):
|
||||
|
||||
def test_location_successful_parsing(self):
|
||||
|
||||
@@ -111,7 +111,7 @@ class TestManagedStream(BlobExchangeTestBase):
|
||||
mock_node = mock.Mock(spec=Node)
|
||||
q = asyncio.Queue()
|
||||
|
||||
bad_peer = make_kademlia_peer(b'2' * 48, "127.0.0.1", tcp_port=3334)
|
||||
bad_peer = make_kademlia_peer(b'2' * 48, "127.0.0.1", tcp_port=3334, allow_localhost=True)
|
||||
|
||||
def _mock_accumulate_peers(q1, q2):
|
||||
async def _task():
|
||||
|
||||
@@ -10,7 +10,7 @@ from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase
|
||||
from tests.unit.lbrynet_daemon.test_ExchangeRateManager import get_dummy_exchange_rate_manager
|
||||
from lbry.utils import generate_id
|
||||
from torba.client.errors import InsufficientFundsError
|
||||
from lbry.error import KeyFeeAboveMaxAllowed, ResolveError, DownloadSDTimeout, DownloadDataTimeout
|
||||
from lbry.error import KeyFeeAboveMaxAllowedError, ResolveError, DownloadSDTimeoutError, DownloadDataTimeoutError
|
||||
from torba.client.wallet import Wallet
|
||||
from torba.client.constants import CENT, NULL_HASH32
|
||||
from torba.client.basenetwork import ClientSession
|
||||
@@ -214,10 +214,10 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
self.assertFalse(event['properties']['added_fixed_peers'])
|
||||
self.assertEqual(event['properties']['connection_failures_count'], 1)
|
||||
self.assertEqual(
|
||||
event['properties']['error_message'], f'Failed to download sd blob {self.sd_hash} within timeout'
|
||||
event['properties']['error_message'], f'Failed to download sd blob {self.sd_hash} within timeout.'
|
||||
)
|
||||
|
||||
await self._test_time_to_first_bytes(check_post, DownloadSDTimeout, after_setup=after_setup)
|
||||
await self._test_time_to_first_bytes(check_post, DownloadSDTimeoutError, after_setup=after_setup)
|
||||
|
||||
async def test_override_fixed_peer_delay_dht_disabled(self):
|
||||
self.client_config.reflector_servers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)]
|
||||
@@ -251,18 +251,18 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
|
||||
def check_post(event):
|
||||
self.assertEqual(event['event'], 'Time To First Bytes')
|
||||
self.assertEqual(event['properties']['error'], 'DownloadSDTimeout')
|
||||
self.assertEqual(event['properties']['error'], 'DownloadSDTimeoutError')
|
||||
self.assertEqual(event['properties']['tried_peers_count'], 0)
|
||||
self.assertEqual(event['properties']['active_peer_count'], 0)
|
||||
self.assertFalse(event['properties']['use_fixed_peers'])
|
||||
self.assertFalse(event['properties']['added_fixed_peers'])
|
||||
self.assertIsNone(event['properties']['fixed_peer_delay'])
|
||||
self.assertEqual(
|
||||
event['properties']['error_message'], f'Failed to download sd blob {self.sd_hash} within timeout'
|
||||
event['properties']['error_message'], f'Failed to download sd blob {self.sd_hash} within timeout.'
|
||||
)
|
||||
|
||||
start = self.loop.time()
|
||||
await self._test_time_to_first_bytes(check_post, DownloadSDTimeout)
|
||||
await self._test_time_to_first_bytes(check_post, DownloadSDTimeoutError)
|
||||
duration = self.loop.time() - start
|
||||
self.assertLessEqual(duration, 5)
|
||||
self.assertGreaterEqual(duration, 3.0)
|
||||
@@ -360,7 +360,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
'version': '_0_0_1'
|
||||
}
|
||||
await self.setup_stream_manager(1000000.0, fee)
|
||||
await self._test_download_error_on_start(KeyFeeAboveMaxAllowed, "")
|
||||
await self._test_download_error_on_start(KeyFeeAboveMaxAllowedError, "")
|
||||
|
||||
async def test_resolve_error(self):
|
||||
await self.setup_stream_manager()
|
||||
@@ -371,7 +371,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
self.server.stop_server()
|
||||
await self.setup_stream_manager()
|
||||
await self._test_download_error_analytics_on_start(
|
||||
DownloadSDTimeout, f'Failed to download sd blob {self.sd_hash} within timeout', timeout=1
|
||||
DownloadSDTimeoutError, f'Failed to download sd blob {self.sd_hash} within timeout.', timeout=1
|
||||
)
|
||||
|
||||
async def test_download_data_timeout(self):
|
||||
@@ -380,7 +380,7 @@ class TestStreamManager(BlobExchangeTestBase):
|
||||
head_blob_hash = json.loads(sdf.read())['blobs'][0]['blob_hash']
|
||||
self.server_blob_manager.delete_blob(head_blob_hash)
|
||||
await self._test_download_error_analytics_on_start(
|
||||
DownloadDataTimeout, f'Failed to download data blobs for sd hash {self.sd_hash} within timeout', timeout=1
|
||||
DownloadDataTimeoutError, f'Failed to download data blobs for sd hash {self.sd_hash} within timeout.', timeout=1
|
||||
)
|
||||
|
||||
async def test_unexpected_error(self):
|
||||
|
||||
@@ -28,14 +28,6 @@ def get_tx():
|
||||
return Transaction().add_inputs([get_input()])
|
||||
|
||||
|
||||
class OldWalletServerTransaction:
|
||||
def __init__(self, tx):
|
||||
self.tx = tx
|
||||
|
||||
def serialize(self):
|
||||
return self.tx.raw
|
||||
|
||||
|
||||
class TestSQLDB(unittest.TestCase):
|
||||
query_timeout = 0.25
|
||||
|
||||
@@ -58,7 +50,7 @@ class TestSQLDB(unittest.TestCase):
|
||||
if txi is not None:
|
||||
tx.add_inputs([txi])
|
||||
self._txos[output.ref.hash] = output
|
||||
return OldWalletServerTransaction(tx), tx.hash
|
||||
return tx, tx.hash
|
||||
|
||||
def _set_channel_key(self, channel, key):
|
||||
private_key = ecdsa.SigningKey.from_string(key*32, curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
|
||||
@@ -87,12 +79,12 @@ class TestSQLDB(unittest.TestCase):
|
||||
claim.stream.title = title
|
||||
result = self._make_tx(Output.pay_claim_name_pubkey_hash(amount, name, claim, b'abc'))
|
||||
if channel:
|
||||
result[0].tx.outputs[0].sign(channel)
|
||||
result[0].tx._reset()
|
||||
result[0].outputs[0].sign(channel)
|
||||
result[0]._reset()
|
||||
return result
|
||||
|
||||
def get_stream_update(self, tx, amount, channel=None):
|
||||
stream = Transaction(tx[0].serialize()).outputs[0]
|
||||
stream = Transaction(tx[0].raw).outputs[0]
|
||||
result = self._make_tx(
|
||||
Output.pay_update_claim_pubkey_hash(
|
||||
amount, stream.claim_name, stream.claim_id, stream.claim, b'abc'
|
||||
@@ -100,19 +92,19 @@ class TestSQLDB(unittest.TestCase):
|
||||
Input.spend(stream)
|
||||
)
|
||||
if channel:
|
||||
result[0].tx.outputs[0].sign(channel)
|
||||
result[0].tx._reset()
|
||||
result[0].outputs[0].sign(channel)
|
||||
result[0]._reset()
|
||||
return result
|
||||
|
||||
def get_abandon(self, tx):
|
||||
claim = Transaction(tx[0].serialize()).outputs[0]
|
||||
claim = Transaction(tx[0].raw).outputs[0]
|
||||
return self._make_tx(
|
||||
Output.pay_pubkey_hash(claim.amount, b'abc'),
|
||||
Input.spend(claim)
|
||||
)
|
||||
|
||||
def get_support(self, tx, amount):
|
||||
claim = Transaction(tx[0].serialize()).outputs[0]
|
||||
claim = Transaction(tx[0].raw).outputs[0]
|
||||
return self._make_tx(
|
||||
Output.pay_support_pubkey_hash(
|
||||
amount, claim.claim_name, claim.claim_id, b'abc'
|
||||
@@ -147,7 +139,7 @@ class TestSQLDB(unittest.TestCase):
|
||||
def advance(self, height, txs):
|
||||
self._current_height = height
|
||||
self.sql.advance_txs(height, txs, {'timestamp': 1}, self.daemon_height, self.timer)
|
||||
return [otx[0].tx.outputs[0] for otx in txs]
|
||||
return [otx[0].outputs[0] for otx in txs]
|
||||
|
||||
def state(self, controlling=None, active=None, accepted=None):
|
||||
self.assertEqual(controlling, self.get_controlling())
|
||||
@@ -328,7 +320,7 @@ class TestClaimtrie(TestSQLDB):
|
||||
iterations = cached_iteration+1 if cached_iteration else 100
|
||||
for i in range(cached_iteration or 1, iterations):
|
||||
stream = getter(f'claim #{i}', COIN, **kwargs)
|
||||
if stream[0].tx.outputs[0].claim_id.startswith(prefix):
|
||||
if stream[0].outputs[0].claim_id.startswith(prefix):
|
||||
cached_iteration is None and print(f'Found "{prefix}" in {i} iterations.')
|
||||
return stream
|
||||
if cached_iteration:
|
||||
@@ -346,8 +338,8 @@ class TestClaimtrie(TestSQLDB):
|
||||
|
||||
tx_chan_a = self.get_channel_with_claim_id_prefix('a', 1, key=b'c')
|
||||
tx_chan_ab = self.get_channel_with_claim_id_prefix('ab', 72, key=b'c')
|
||||
txo_chan_a = tx_chan_a[0].tx.outputs[0]
|
||||
txo_chan_ab = tx_chan_ab[0].tx.outputs[0]
|
||||
txo_chan_a = tx_chan_a[0].outputs[0]
|
||||
txo_chan_ab = tx_chan_ab[0].outputs[0]
|
||||
advance(1, [tx_chan_a])
|
||||
advance(2, [tx_chan_ab])
|
||||
r_ab, r_a = reader._search(order_by=['creation_height'], limit=2)
|
||||
@@ -373,8 +365,8 @@ class TestClaimtrie(TestSQLDB):
|
||||
|
||||
tx_a2 = self.get_stream_with_claim_id_prefix('a', 7, channel=txo_chan_a)
|
||||
tx_ab2 = self.get_stream_with_claim_id_prefix('ab', 23, channel=txo_chan_a)
|
||||
a2_claim = tx_a2[0].tx.outputs[0]
|
||||
ab2_claim = tx_ab2[0].tx.outputs[0]
|
||||
a2_claim = tx_a2[0].outputs[0]
|
||||
ab2_claim = tx_ab2[0].outputs[0]
|
||||
advance(6, [tx_a2])
|
||||
advance(7, [tx_ab2])
|
||||
r_ab2, r_a2 = reader._search(order_by=['creation_height'], limit=2)
|
||||
@@ -430,8 +422,8 @@ class TestClaimtrie(TestSQLDB):
|
||||
|
||||
tx_chan_a = self.get_channel_with_claim_id_prefix('a', 1, key=b'c')
|
||||
tx_chan_ab = self.get_channel_with_claim_id_prefix('ab', 72, key=b'c')
|
||||
txo_chan_a = tx_chan_a[0].tx.outputs[0]
|
||||
txo_chan_ab = tx_chan_ab[0].tx.outputs[0]
|
||||
txo_chan_a = tx_chan_a[0].outputs[0]
|
||||
txo_chan_ab = tx_chan_ab[0].outputs[0]
|
||||
advance(1, [tx_chan_a])
|
||||
advance(2, [tx_chan_ab])
|
||||
|
||||
@@ -485,3 +477,9 @@ class TestTrending(TestSQLDB):
|
||||
self.assertEqual([53, 38, -32, 0, -6], [int(c['trending_global']) for c in results])
|
||||
self.assertEqual([4, 4, 2, 0, 1], [int(c['trending_group']) for c in results])
|
||||
self.assertEqual([53, 38, 2, 0, -6], [int(c['trending_mixed']) for c in results])
|
||||
|
||||
def test_edge(self):
|
||||
problematic = self.get_stream('Problem', COIN)
|
||||
self.advance(1, [problematic])
|
||||
self.advance(TRENDING_WINDOW, [self.get_support(problematic, 53000000000)])
|
||||
self.advance(TRENDING_WINDOW * 2, [self.get_support(problematic, 500000000)])
|
||||
+1
-1
@@ -13,4 +13,4 @@ setenv =
|
||||
TORBA_LEDGER=lbry.wallet
|
||||
commands =
|
||||
orchstr8 download
|
||||
coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration
|
||||
coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration {posargs}
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/torba/master/torba.png" alt="Torba" width="42" height="30" /> Torba [](https://travis-ci.org/lbryio/torba) [](https://codecov.io/gh/lbryio/torba)
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/torba/master/torba.png" alt="Torba" width="42" height="30" /> Torba [](https://travis-ci.org/lbryio/torba)
|
||||
|
||||
A new wallet library to help bitcoin based projects build fast, correct and scalable crypto currency wallets in Python.
|
||||
@@ -74,6 +74,21 @@ class ReconnectTests(IntegrationTestCase):
|
||||
with self.assertRaises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(switch_event, timeout=1)
|
||||
|
||||
async def test_direct_sync(self):
|
||||
await self.ledger.stop()
|
||||
initial_height = self.ledger.local_height_including_downloaded_height
|
||||
await self.blockchain.generate(100)
|
||||
while self.conductor.spv_node.server.bp.height < initial_height + 99: # off by 1
|
||||
await asyncio.sleep(0.1)
|
||||
self.assertEqual(initial_height, self.ledger.local_height_including_downloaded_height)
|
||||
# locks header processing so we make sure we are the only ones modifying it
|
||||
async with self.ledger._header_processing_lock:
|
||||
await self.ledger.headers.open()
|
||||
await self.ledger.network.start()
|
||||
await self.ledger.network.on_connected.first
|
||||
await self.ledger.initial_headers_sync()
|
||||
self.assertEqual(initial_height + 100, self.ledger.local_height_including_downloaded_height)
|
||||
|
||||
async def test_connection_drop_still_receives_events_after_reconnected(self):
|
||||
address1 = await self.account.receiving.get_or_create_usable_address()
|
||||
# disconnect and send a new tx, should reconnect and get it
|
||||
|
||||
@@ -175,5 +175,3 @@ class BasicTransactionTests(IntegrationTestCase):
|
||||
self.assertTrue(await self.ledger.update_history(address, remote_status))
|
||||
self.assertEqual(21, len((await self.ledger.get_local_status_and_history(address))[1]))
|
||||
self.assertEqual(0, len(self.ledger._known_addresses_out_of_sync))
|
||||
# should be another test, but it would be too much to setup just for that and it affects sync
|
||||
self.assertIsNone(await self.ledger.network.retriable_call(self.ledger.network.get_transaction, '1'*64))
|
||||
@@ -1,8 +1,9 @@
|
||||
import asyncio
|
||||
import os
|
||||
import tempfile
|
||||
from urllib.request import Request, urlopen
|
||||
from binascii import hexlify
|
||||
|
||||
from torba.client.hash import sha256
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
from torba.coin.bitcoinsegwit import MainHeaders
|
||||
@@ -125,6 +126,21 @@ class BasicHeadersTests(BitcoinHeadersTestCase):
|
||||
await headers.connect(len(headers), self.get_bytes(block_bytes(3001 - 1500), after=block_bytes(1500)))
|
||||
self.assertEqual(headers.height, 3000)
|
||||
|
||||
async def test_checkpointed_writer(self):
|
||||
headers = MainHeaders(':memory:')
|
||||
headers.checkpoint = 100, hexlify(sha256(self.get_bytes(block_bytes(100))))
|
||||
genblocks = lambda start, end: self.get_bytes(block_bytes(end - start), block_bytes(start))
|
||||
async with headers.checkpointed_connector() as buff:
|
||||
buff.write(genblocks(0, 10))
|
||||
self.assertEqual(len(headers), 10)
|
||||
async with headers.checkpointed_connector() as buff:
|
||||
buff.write(genblocks(10, 100))
|
||||
self.assertEqual(len(headers), 100)
|
||||
headers = MainHeaders(':memory:')
|
||||
async with headers.checkpointed_connector() as buff:
|
||||
buff.write(genblocks(0, 300))
|
||||
self.assertEqual(len(headers), 300)
|
||||
|
||||
async def test_concurrency(self):
|
||||
BLOCKS = 30
|
||||
headers_temporary_file = tempfile.mktemp()
|
||||
|
||||
@@ -11,6 +11,7 @@ from torba.client.basetransaction import BaseTransaction, TXRefImmutable
|
||||
from torba.client.bip32 import PubKey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
sqlite3.enable_callback_tracebacks(True)
|
||||
|
||||
|
||||
class AIOSQLite:
|
||||
@@ -24,6 +25,7 @@ class AIOSQLite:
|
||||
|
||||
@classmethod
|
||||
async def connect(cls, path: Union[bytes, str], *args, **kwargs):
|
||||
sqlite3.enable_callback_tracebacks(True)
|
||||
def _connect():
|
||||
return sqlite3.connect(path, *args, **kwargs)
|
||||
db = cls()
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from io import BytesIO
|
||||
from typing import Optional, Iterator, Tuple
|
||||
from binascii import hexlify
|
||||
@@ -28,6 +31,7 @@ class BaseHeaders:
|
||||
target_timespan: int
|
||||
|
||||
validate_difficulty: bool = True
|
||||
checkpoint = None
|
||||
|
||||
def __init__(self, path) -> None:
|
||||
if path == ':memory:':
|
||||
@@ -99,6 +103,33 @@ class BaseHeaders:
|
||||
return b'0' * 64
|
||||
return hexlify(double_sha256(header)[::-1])
|
||||
|
||||
@asynccontextmanager
|
||||
async def checkpointed_connector(self):
|
||||
buf = BytesIO()
|
||||
try:
|
||||
yield buf
|
||||
finally:
|
||||
await asyncio.sleep(0)
|
||||
final_height = len(self) + buf.tell() // self.header_size
|
||||
verifiable_bytes = (self.checkpoint[0] - len(self)) * self.header_size if self.checkpoint else 0
|
||||
if verifiable_bytes > 0 and final_height >= self.checkpoint[0]:
|
||||
buf.seek(0)
|
||||
self.io.seek(0)
|
||||
h = hashlib.sha256()
|
||||
h.update(self.io.read())
|
||||
h.update(buf.read(verifiable_bytes))
|
||||
if h.hexdigest().encode() == self.checkpoint[1]:
|
||||
buf.seek(0)
|
||||
self._write(len(self), buf.read(verifiable_bytes))
|
||||
remaining = buf.read()
|
||||
buf.seek(0)
|
||||
buf.write(remaining)
|
||||
buf.truncate()
|
||||
else:
|
||||
log.warning("Checkpoint mismatch, connecting headers through slow method.")
|
||||
if buf.tell() > 0:
|
||||
await self.connect(len(self), buf.getvalue())
|
||||
|
||||
async def connect(self, start: int, headers: bytes) -> int:
|
||||
added = 0
|
||||
bail = False
|
||||
@@ -109,20 +140,21 @@ class BaseHeaders:
|
||||
except InvalidHeader as e:
|
||||
bail = True
|
||||
chunk = chunk[:(height-e.height)*self.header_size]
|
||||
written = 0
|
||||
if chunk:
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
written = self.io.write(chunk) // self.header_size
|
||||
self.io.truncate()
|
||||
# .seek()/.write()/.truncate() might also .flush() when needed
|
||||
# the goal here is mainly to ensure we're definitely flush()'ing
|
||||
self.io.flush()
|
||||
self._size = self.io.tell() // self.header_size
|
||||
added += written
|
||||
added += self._write(height, chunk) if chunk else 0
|
||||
if bail:
|
||||
break
|
||||
return added
|
||||
|
||||
def _write(self, height, verified_chunk):
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
written = self.io.write(verified_chunk) // self.header_size
|
||||
self.io.truncate()
|
||||
# .seek()/.write()/.truncate() might also .flush() when needed
|
||||
# the goal here is mainly to ensure we're definitely flush()'ing
|
||||
self.io.flush()
|
||||
self._size = self.io.tell() // self.header_size
|
||||
return written
|
||||
|
||||
def validate_chunk(self, height, chunk):
|
||||
previous_hash, previous_header, previous_previous_header = None, None, None
|
||||
if height > 0:
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import base64
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import zlib
|
||||
from functools import partial
|
||||
from binascii import hexlify, unhexlify
|
||||
from io import StringIO
|
||||
@@ -136,6 +138,7 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
self.get_id(), change, self.headers.height
|
||||
)
|
||||
)
|
||||
self._download_height = 0
|
||||
|
||||
self._on_ready_controller = StreamController()
|
||||
self.on_ready = self._on_ready_controller.stream
|
||||
@@ -285,6 +288,8 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
first_connection = self.network.on_connected.first
|
||||
asyncio.ensure_future(self.network.start())
|
||||
await first_connection
|
||||
async with self._header_processing_lock:
|
||||
await self.initial_headers_sync()
|
||||
await self.join_network()
|
||||
self.network.on_connected.listen(self.join_network)
|
||||
|
||||
@@ -303,6 +308,26 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
await self.db.close()
|
||||
await self.headers.close()
|
||||
|
||||
@property
|
||||
def local_height_including_downloaded_height(self):
|
||||
return max(self.headers.height, self._download_height)
|
||||
|
||||
async def initial_headers_sync(self):
|
||||
target = self.network.remote_height + 1
|
||||
current = len(self.headers)
|
||||
get_chunk = partial(self.network.retriable_call, self.network.get_headers, count=4096, b64=True)
|
||||
chunks = [asyncio.create_task(get_chunk(height)) for height in range(current, target, 4096)]
|
||||
total = 0
|
||||
async with self.headers.checkpointed_connector() as buffer:
|
||||
for chunk in chunks:
|
||||
headers = await chunk
|
||||
total += len(headers['base64'])
|
||||
buffer.write(
|
||||
zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
|
||||
)
|
||||
self._download_height = current + buffer.tell() // self.headers.header_size
|
||||
log.info("Headers sync: %s / %s", self._download_height, target)
|
||||
|
||||
async def update_headers(self, height=None, headers=None, subscription_update=False):
|
||||
rewound = 0
|
||||
while True:
|
||||
@@ -525,12 +550,8 @@ class BaseLedger(metaclass=LedgerRegistry):
|
||||
if tx is None:
|
||||
# fetch from network
|
||||
_raw = await self.network.retriable_call(self.network.get_transaction, txid, remote_height)
|
||||
if _raw:
|
||||
tx = self.transaction_class(unhexlify(_raw))
|
||||
cache_item.tx = tx # make sure it's saved before caching it
|
||||
|
||||
if tx is None:
|
||||
raise ValueError(f'Transaction {txid} was not in database and not on network.')
|
||||
tx = self.transaction_class(unhexlify(_raw))
|
||||
cache_item.tx = tx # make sure it's saved before caching it
|
||||
|
||||
await self.maybe_verify_transaction(tx, remote_height)
|
||||
return tx
|
||||
|
||||
@@ -72,9 +72,6 @@ class ClientSession(BaseClientSession):
|
||||
log.debug("got reply for %s from %s:%i", method, *self.server)
|
||||
return reply
|
||||
except (RPCError, ProtocolError) as e:
|
||||
if str(e).find('.*no such .*transaction.*'):
|
||||
# shouldn't the server return none instead?
|
||||
return None
|
||||
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
|
||||
*self.server, *e.args)
|
||||
raise e
|
||||
@@ -105,6 +102,9 @@ class ClientSession(BaseClientSession):
|
||||
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
|
||||
await self.ensure_server_version()
|
||||
retry_delay = default_delay
|
||||
except RPCError as e:
|
||||
log.warning("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
|
||||
retry_delay = 60 * 60
|
||||
except (asyncio.TimeoutError, OSError):
|
||||
await self.close()
|
||||
retry_delay = min(60, retry_delay * 2)
|
||||
@@ -116,7 +116,8 @@ class ClientSession(BaseClientSession):
|
||||
finally:
|
||||
self.trigger_urgent_reconnect.clear()
|
||||
|
||||
async def ensure_server_version(self, required='1.2', timeout=3):
|
||||
async def ensure_server_version(self, required=None, timeout=3):
|
||||
required = required or self.network.PROTOCOL_VERSION
|
||||
return await asyncio.wait_for(
|
||||
self.send_request('server.version', [__version__, required]), timeout=timeout
|
||||
)
|
||||
@@ -142,6 +143,7 @@ class ClientSession(BaseClientSession):
|
||||
|
||||
|
||||
class BaseNetwork:
|
||||
PROTOCOL_VERSION = '1.2'
|
||||
|
||||
def __init__(self, ledger):
|
||||
self.config = ledger.config
|
||||
@@ -173,9 +175,9 @@ class BaseNetwork:
|
||||
continue
|
||||
self.client = await self.session_pool.wait_for_fastest_session()
|
||||
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
|
||||
self._on_connected_controller.add(True)
|
||||
try:
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
log.info("Subscribed to headers: %s:%d", *self.client.server)
|
||||
except (asyncio.TimeoutError, ConnectionError):
|
||||
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
|
||||
@@ -239,8 +241,9 @@ class BaseNetwork:
|
||||
restricted = 0 > height > self.remote_height - 10
|
||||
return self.rpc('blockchain.transaction.get_merkle', [tx_hash, height], restricted)
|
||||
|
||||
def get_headers(self, height, count=10000):
|
||||
return self.rpc('blockchain.block.headers', [height, count])
|
||||
def get_headers(self, height, count=10000, b64=False):
|
||||
restricted = height >= self.remote_height - 100
|
||||
return self.rpc('blockchain.block.headers', [height, count, 0, b64], restricted)
|
||||
|
||||
# --- Subscribes, history and broadcasts are always aimed towards the master client directly
|
||||
def get_history(self, address):
|
||||
|
||||
@@ -205,7 +205,7 @@ class SPVNode:
|
||||
self.session_timeout = 600
|
||||
self.rpc_port = '0' # disabled by default
|
||||
|
||||
async def start(self, blockchain_node: 'BlockchainNode'):
|
||||
async def start(self, blockchain_node: 'BlockchainNode', extraconf=None):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
conf = {
|
||||
'DB_DIRECTORY': self.data_path,
|
||||
@@ -218,6 +218,8 @@ class SPVNode:
|
||||
'INDIVIDUAL_TAG_INDEXES': '',
|
||||
'RPC_PORT': self.rpc_port
|
||||
}
|
||||
if extraconf:
|
||||
conf.update(extraconf)
|
||||
# TODO: don't use os.environ
|
||||
os.environ.update(conf)
|
||||
self.server = Server(Env(self.coin_class))
|
||||
|
||||
@@ -645,12 +645,17 @@ class BlockProcessor:
|
||||
could be lost.
|
||||
"""
|
||||
self._caught_up_event = caught_up_event
|
||||
await self._first_open_dbs()
|
||||
try:
|
||||
await self._first_open_dbs()
|
||||
await asyncio.wait([
|
||||
self.prefetcher.main_loop(self.height),
|
||||
self._process_prefetched_blocks()
|
||||
])
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except:
|
||||
self.logger.exception("Block processing failed!")
|
||||
raise
|
||||
finally:
|
||||
# Shut down block processing
|
||||
self.logger.info('flushing to DB for a clean shutdown...')
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
# and warranty status of this software.
|
||||
|
||||
"""Classes for local RPC server and remote client TCP/SSL servers."""
|
||||
import base64
|
||||
import collections
|
||||
import asyncio
|
||||
import codecs
|
||||
@@ -13,6 +14,8 @@ import datetime
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import zlib
|
||||
|
||||
import pylru
|
||||
import ssl
|
||||
import time
|
||||
@@ -615,7 +618,7 @@ class SessionBase(RPCSession):
|
||||
sessions.
|
||||
"""
|
||||
|
||||
MAX_CHUNK_SIZE = 2016
|
||||
MAX_CHUNK_SIZE = 40960
|
||||
session_counter = itertools.count()
|
||||
request_handlers: typing.Dict[str, typing.Callable] = {}
|
||||
version = '0.5.7'
|
||||
@@ -1007,7 +1010,7 @@ class ElectrumX(SessionBase):
|
||||
height: the header's height"""
|
||||
return await self.block_header(height)
|
||||
|
||||
async def block_headers(self, start_height, count, cp_height=0):
|
||||
async def block_headers(self, start_height, count, cp_height=0, b64=False):
|
||||
"""Return count concatenated block headers as hex for the main chain;
|
||||
starting at start_height.
|
||||
|
||||
@@ -1021,15 +1024,18 @@ class ElectrumX(SessionBase):
|
||||
max_size = self.MAX_CHUNK_SIZE
|
||||
count = min(count, max_size)
|
||||
headers, count = await self.db.read_headers(start_height, count)
|
||||
result = {'hex': headers.hex(), 'count': count, 'max': max_size}
|
||||
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
|
||||
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() if b64 else headers.hex()
|
||||
result = {
|
||||
'base64' if b64 else 'hex': headers,
|
||||
'count': count,
|
||||
'max': max_size
|
||||
}
|
||||
if count and cp_height:
|
||||
last_height = start_height + count - 1
|
||||
result.update(await self._merkle_proof(cp_height, last_height))
|
||||
return result
|
||||
|
||||
async def block_headers_12(self, start_height, count):
|
||||
return await self.block_headers(start_height, count)
|
||||
|
||||
async def block_get_chunk(self, index):
|
||||
"""Return a chunk of block headers as a hexadecimal string.
|
||||
|
||||
@@ -1270,7 +1276,7 @@ class ElectrumX(SessionBase):
|
||||
handlers.update({
|
||||
'mempool.get_fee_histogram':
|
||||
self.mempool.compact_fee_histogram,
|
||||
'blockchain.block.headers': self.block_headers_12,
|
||||
'blockchain.block.headers': self.block_headers,
|
||||
'server.ping': self.ping,
|
||||
})
|
||||
|
||||
|
||||
+10
-33
@@ -41,19 +41,9 @@ ZERO = bytes(32)
|
||||
MINUS_1 = 4294967295
|
||||
|
||||
|
||||
class Tx(namedtuple("Tx", "version inputs outputs locktime")):
|
||||
class Tx(namedtuple("Tx", "version inputs outputs locktime raw")):
|
||||
"""Class representing a transaction."""
|
||||
|
||||
def serialize(self):
|
||||
return b''.join((
|
||||
pack_le_int32(self.version),
|
||||
pack_varint(len(self.inputs)),
|
||||
b''.join(tx_in.serialize() for tx_in in self.inputs),
|
||||
pack_varint(len(self.outputs)),
|
||||
b''.join(tx_out.serialize() for tx_out in self.outputs),
|
||||
pack_le_uint32(self.locktime)
|
||||
))
|
||||
|
||||
|
||||
class TxInput(namedtuple("TxInput", "prev_hash prev_idx script sequence")):
|
||||
"""Class representing a transaction input."""
|
||||
@@ -105,22 +95,13 @@ class Deserializer:
|
||||
|
||||
def read_tx(self):
|
||||
"""Return a deserialized transaction."""
|
||||
version = self._read_le_int32()
|
||||
inputs = self._read_inputs()
|
||||
outputs = self._read_outputs()
|
||||
if self.flags == 1:
|
||||
# drain witness portion of transaction
|
||||
# too many witnesses for no crime
|
||||
for i in range(len(inputs)):
|
||||
for v in range(self._read_varint()):
|
||||
self._read_varbytes()
|
||||
self.flags = 0
|
||||
locktime = self._read_le_uint32()
|
||||
start = self.cursor
|
||||
return Tx(
|
||||
version,
|
||||
inputs,
|
||||
outputs,
|
||||
locktime
|
||||
self._read_le_int32(), # version
|
||||
self._read_inputs(), # inputs
|
||||
self._read_outputs(), # outputs
|
||||
self._read_le_uint32(), # locktime
|
||||
self.binary[start:self.cursor],
|
||||
)
|
||||
|
||||
def read_tx_and_hash(self):
|
||||
@@ -144,11 +125,7 @@ class Deserializer:
|
||||
|
||||
def _read_inputs(self):
|
||||
read_input = self._read_input
|
||||
num_inputs = self._read_varint()
|
||||
if num_inputs == 0:
|
||||
self.flags = self._read_byte()
|
||||
num_inputs = self._read_varint()
|
||||
return [read_input() for i in range(num_inputs)]
|
||||
return [read_input() for i in range(self._read_varint())]
|
||||
|
||||
def _read_input(self):
|
||||
return TxInput(
|
||||
@@ -220,7 +197,7 @@ class Deserializer:
|
||||
|
||||
|
||||
class TxSegWit(namedtuple("Tx", "version marker flag inputs outputs "
|
||||
"witness locktime")):
|
||||
"witness locktime raw")):
|
||||
"""Class representing a SegWit transaction."""
|
||||
|
||||
|
||||
@@ -266,7 +243,7 @@ class DeserializerSegWit(Deserializer):
|
||||
vsize = (3 * base_size + self.binary_length) // 4
|
||||
|
||||
return TxSegWit(version, marker, flag, inputs, outputs, witness,
|
||||
locktime), self.TX_HASH_FN(orig_ser), vsize
|
||||
locktime, orig_ser), self.TX_HASH_FN(orig_ser), vsize
|
||||
|
||||
def read_tx(self):
|
||||
return self._read_tx_parts()[0]
|
||||
|
||||
Reference in new issue
Block a user