Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6a67a34eb4 |
No files matched your search
@@ -0,0 +1,10 @@
|
||||
[bumpversion]
|
||||
current_version = 0.3.8
|
||||
commit = True
|
||||
tag = True
|
||||
message = Bump version: {current_version} -> {new_version}
|
||||
|
||||
[bumpversion:file:lbrynet/__init__.py]
|
||||
|
||||
[bumpversion:file:packaging/ubuntu/lbry.desktop]
|
||||
|
||||
@@ -1,197 +0,0 @@
|
||||
name: ci
|
||||
on: push
|
||||
jobs:
|
||||
|
||||
lint:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: |
|
||||
pip install --user --upgrade pip wheel
|
||||
pip install -e .[lint]
|
||||
- run: make lint
|
||||
|
||||
tests-unit:
|
||||
name: "tests / unit"
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- name: set pip cache dir
|
||||
id: pip-cache
|
||||
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- run: |
|
||||
pip install --user --upgrade pip wheel
|
||||
pip install -e .[test]
|
||||
- env:
|
||||
HOME: /tmp
|
||||
run: coverage run -m unittest discover -v tests.unit
|
||||
- env:
|
||||
COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
COVERALLS_PARALLEL: true
|
||||
name: Submit to coveralls
|
||||
run: |
|
||||
pip install https://github.com/bboe/coveralls-python/archive/github_actions.zip
|
||||
coveralls
|
||||
|
||||
tests-integration:
|
||||
name: "tests / integration"
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
test:
|
||||
# - datanetwork
|
||||
- blockchain
|
||||
# - other
|
||||
db:
|
||||
- sqlite
|
||||
- postgres
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:12
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: postgres
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- if: matrix.test == 'other'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends ffmpeg
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./.tox
|
||||
key: tox-integration-${{ matrix.test }}-${{ matrix.db }}-${{ hashFiles('setup.py') }}
|
||||
restore-keys: txo-integration-${{ matrix.test }}-${{ matrix.db }}-
|
||||
- run: pip install tox
|
||||
- env:
|
||||
TEST_DB: ${{ matrix.db }}
|
||||
run: tox -e ${{ matrix.test }}
|
||||
- env:
|
||||
COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
COVERALLS_PARALLEL: true
|
||||
name: Submit to coveralls
|
||||
run: |
|
||||
pip install https://github.com/bboe/coveralls-python/archive/github_actions.zip
|
||||
coverage combine tests
|
||||
coveralls
|
||||
|
||||
coveralls-finished:
|
||||
needs: ["tests-unit", "tests-integration"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Coveralls Finished
|
||||
uses: coverallsapp/github-action@57daa114
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
parallel-finished: true
|
||||
|
||||
build:
|
||||
needs: ["lint", "tests-unit", "tests-integration"]
|
||||
name: "build"
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-16.04
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- name: set pip cache dir
|
||||
id: pip-cache
|
||||
run: echo "::set-output name=dir::$(pip cache dir)"
|
||||
- name: extract pip cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ steps.pip-cache.outputs.dir }}
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
|
||||
restore-keys: ${{ runner.os }}-pip-
|
||||
- name: Setup
|
||||
run: |
|
||||
pip install --user --upgrade pip wheel
|
||||
pip install pyinstaller@git+https://github.com/eukreign/pyinstaller.git@sqlalchemy
|
||||
- if: startsWith(runner.os, 'linux')
|
||||
run: |
|
||||
sudo apt-get install libzmq3-dev
|
||||
pip install -e .[postgres]
|
||||
- if: startsWith(runner.os, 'mac')
|
||||
run: |
|
||||
brew install zeromq
|
||||
pip install -e .
|
||||
- if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac')
|
||||
name: Build & Run (Unix)
|
||||
run: |
|
||||
pyinstaller --onefile --name lbrynet lbry/cli.py
|
||||
chmod +x dist/lbrynet
|
||||
dist/lbrynet --version
|
||||
- if: startsWith(runner.os, 'windows')
|
||||
name: Build & Run (Windows)
|
||||
run: |
|
||||
pip install pywin32
|
||||
pip install -e .
|
||||
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/cli.py
|
||||
dist/lbrynet.exe --version
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: lbrynet-${{ matrix.os }}
|
||||
path: dist/
|
||||
|
||||
docker:
|
||||
needs: ["build"]
|
||||
name: "build (docker)"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: fetch lbrynet binary
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: lbrynet-ubuntu-16.04
|
||||
- run: |
|
||||
chmod +x lbrynet
|
||||
mv lbrynet docker
|
||||
- name: build and push docker image
|
||||
uses: docker/build-push-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
repository: lbry/lbrynet
|
||||
path: docker
|
||||
tag_with_ref: true
|
||||
tag_with_sha: true
|
||||
add_git_labels: true
|
||||
+22
-13
@@ -1,18 +1,27 @@
|
||||
/.idea
|
||||
/.DS_Store
|
||||
*.pyc
|
||||
|
||||
*.xml
|
||||
|
||||
.idea/.name
|
||||
|
||||
*.iml
|
||||
|
||||
*.log
|
||||
|
||||
*.txt
|
||||
|
||||
lbrynet.egg-info/PKG-INFO
|
||||
|
||||
*.egg
|
||||
|
||||
/build
|
||||
|
||||
/dist
|
||||
/.tox
|
||||
/.coverage*
|
||||
/lbry-venv
|
||||
/venv
|
||||
|
||||
lbry.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
*.so
|
||||
|
||||
/tests/integration/commands/files
|
||||
/tests/.coverage.*
|
||||
*.pem
|
||||
|
||||
/lbry/blockchain/bin
|
||||
/lbry/wallet/bin
|
||||
*.decTest
|
||||
|
||||
.coverage
|
||||
-210
@@ -1,210 +0,0 @@
|
||||
default:
|
||||
image: python:3.7
|
||||
|
||||
|
||||
#cache:
|
||||
# directories:
|
||||
# - $HOME/venv
|
||||
# - $HOME/.cache/pip
|
||||
# - $HOME/Library/Caches/pip
|
||||
# - $HOME/Library/Caches/Homebrew
|
||||
# - $TRAVIS_BUILD_DIR/.tox
|
||||
|
||||
|
||||
stages:
|
||||
- test
|
||||
- build
|
||||
- assets
|
||||
- release
|
||||
|
||||
|
||||
.tagged:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG =~ /^v[0-9\.]+$/'
|
||||
when: on_success
|
||||
|
||||
|
||||
|
||||
test:lint:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- make lint
|
||||
|
||||
test:unit:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
test:datanetwork-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e datanetwork
|
||||
|
||||
test:blockchain-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- tox -e blockchain
|
||||
|
||||
test:other-integration:
|
||||
stage: test
|
||||
script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends ffmpeg
|
||||
- pip install tox-travis
|
||||
- tox -e other
|
||||
|
||||
test:json-api:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
|
||||
|
||||
|
||||
.build:
|
||||
stage: build
|
||||
artifacts:
|
||||
expire_in: 1 day
|
||||
paths:
|
||||
- lbrynet-${OS}.zip
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller
|
||||
- pip install -e .
|
||||
- python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
- chmod +x dist/lbrynet
|
||||
- zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet # gitlab expects artifacts to be in $CI_PROJECT_DIR
|
||||
- openssl dgst -sha256 ${CI_PROJECT_DIR}/lbrynet-${OS}.zip | egrep -o [0-9a-f]+$ # get sha256 of asset. works on mac and ubuntu
|
||||
- dist/lbrynet --version
|
||||
|
||||
build:linux:
|
||||
extends: .build
|
||||
image: ubuntu:16.04
|
||||
variables:
|
||||
OS: linux
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
|
||||
build:mac:
|
||||
extends: .build
|
||||
tags: [macos] # makes gitlab use the mac runner
|
||||
variables:
|
||||
OS: mac
|
||||
GIT_DEPTH: 5
|
||||
VENV: /tmp/gitlab-lbry-sdk-venv
|
||||
before_script:
|
||||
# - brew upgrade python || true
|
||||
- python3 --version | grep -q '^Python 3\.7\.' # dont upgrade python on every run. just make sure we're on the right Python
|
||||
# - pip3 install --user --upgrade pip virtualenv
|
||||
- pip3 --version | grep -q '\(python 3\.7\)'
|
||||
- virtualenv --python=python3.7 "${VENV}"
|
||||
- source "${VENV}/bin/activate"
|
||||
after_script:
|
||||
- rm -rf "${VENV}"
|
||||
|
||||
build:windows:
|
||||
extends: .build
|
||||
tags: [windows] # makes gitlab use the windows runner
|
||||
variables:
|
||||
OS: windows
|
||||
GIT_DEPTH: 5
|
||||
before_script:
|
||||
- ./docker/install_choco.ps1
|
||||
- choco install -y --x86 python3 7zip checksum
|
||||
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
|
||||
- pip --version # | findstr /E '\(python 3\.7\)'
|
||||
- pip install virtualenv pywin32
|
||||
- virtualenv venv
|
||||
- venv/Scripts/activate.ps1
|
||||
- pip install pip==19.3.1; $true # $true ignores errors. need this to get the correct coincurve wheel. see commit notes for details.
|
||||
after_script:
|
||||
- rmdir -Recurse venv
|
||||
script:
|
||||
- pip install --upgrade 'setuptools<45.0.0'
|
||||
- pip install pyinstaller==3.5
|
||||
- pip install -e .
|
||||
- python docker/set_build.py # must come after lbry is installed because it imports lbry
|
||||
- pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
|
||||
- 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
|
||||
- checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip
|
||||
- dist/lbrynet.exe --version
|
||||
|
||||
|
||||
|
||||
# s3 = upload asset to s3 (build.lbry.io)
|
||||
.s3:
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload
|
||||
- pip install awscli
|
||||
- S3_PATH="daemon/gitlab-build-${CI_PIPELINE_ID}_commit-${CI_COMMIT_SHA:0:7}$( if [ ! -z ${CI_COMMIT_TAG} ]; then echo _tag-${CI_COMMIT_TAG}; else echo _branch-${CI_COMMIT_REF_NAME}; fi )"
|
||||
- AWS_ACCESS_KEY_ID=${ARTIFACTS_KEY} AWS_SECRET_ACCESS_KEY=${ARTIFACTS_SECRET} AWS_REGION=${ARTIFACTS_REGION}
|
||||
aws s3 cp lbrynet-${OS}.zip s3://${ARTIFACTS_BUCKET}/${S3_PATH}/lbrynet-${OS}.zip
|
||||
|
||||
s3:linux:
|
||||
extends: .s3
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
s3:mac:
|
||||
extends: .s3
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
s3:windows:
|
||||
extends: .s3
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
# github = upload assets to github when there's a tagged release
|
||||
.github:
|
||||
extends: .tagged
|
||||
stage: assets
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- "[ -f lbrynet-${OS}.zip ]" # check that asset exists before trying to upload. githubrelease won't error if its missing
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} asset lbryio/lbry-sdk upload ${CI_COMMIT_TAG} lbrynet-${OS}.zip
|
||||
|
||||
github:linux:
|
||||
extends: .github
|
||||
variables: {OS: linux}
|
||||
needs: ["build:linux"]
|
||||
|
||||
github:mac:
|
||||
extends: .github
|
||||
variables: {OS: mac}
|
||||
needs: ["build:mac"]
|
||||
|
||||
github:windows:
|
||||
extends: .github
|
||||
variables: {OS: windows}
|
||||
needs: ["build:windows"]
|
||||
|
||||
|
||||
|
||||
publish:
|
||||
extends: .tagged
|
||||
stage: release
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
script:
|
||||
- pip install githubrelease
|
||||
- githubrelease --no-progress --github-token ${GITHUB_CI_USER_ACCESS_TOKEN} release lbryio/lbry-sdk publish ${CI_COMMIT_TAG}
|
||||
- >
|
||||
curl -X POST -H 'Content-type: application/json' --data '{"text":"<!channel> There is a new SDK release: https://github.com/lbryio/lbry-sdk/releases/tag/'"${CI_COMMIT_TAG}"'\n'"$(curl -s "https://api.github.com/repos/lbryio/lbry-sdk/releases/tags/${CI_COMMIT_TAG}" | egrep '\w*\"body\":' | cut -d':' -f 2- | tail -c +3 | head -c -2)"'", "channel":"tech"}' "$(echo ${SLACK_WEBHOOK_URL_BASE64} | base64 -d)"
|
||||
File renamed without changes.
@@ -0,0 +1,379 @@
|
||||
[MASTER]
|
||||
|
||||
# Specify a configuration file.
|
||||
#rcfile=
|
||||
|
||||
# Python code to execute, usually for sys.path manipulation such as
|
||||
# pygtk.require().
|
||||
#init-hook=
|
||||
|
||||
# Add files or directories to the blacklist. They should be base names, not
|
||||
# paths.
|
||||
ignore=CVS
|
||||
|
||||
# Pickle collected data for later comparisons.
|
||||
persistent=yes
|
||||
|
||||
# List of plugins (as comma separated values of python modules names) to load,
|
||||
# usually to register additional checkers.
|
||||
load-plugins=
|
||||
|
||||
# Use multiple processes to speed up Pylint.
|
||||
jobs=1
|
||||
|
||||
# Allow loading of arbitrary C extensions. Extensions are imported into the
|
||||
# active Python interpreter and may run arbitrary code.
|
||||
unsafe-load-any-extension=no
|
||||
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code
|
||||
extension-pkg-whitelist=
|
||||
|
||||
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||
# optimizer, which will apply various small optimizations. For instance, it can
|
||||
# be used to obtain the result of joining multiple strings with the addition
|
||||
# operator. Joining a lot of strings can lead to a maximum recursion error in
|
||||
# Pylint and this flag can prevent that. It has one side effect, the resulting
|
||||
# AST will be different than the one from reality.
|
||||
optimize-ast=no
|
||||
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
|
||||
# Only show warnings with the listed confidence levels. Leave empty to show
|
||||
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
|
||||
confidence=
|
||||
|
||||
# Enable the message, report, category or checker with the given id(s). You can
|
||||
# either give multiple identifier separated by comma (,) or put this option
|
||||
# multiple time (only on the command line, not in the configuration file where
|
||||
# it should appear only once). See also the "--disable" option for examples.
|
||||
#enable=
|
||||
|
||||
# Disable the message, report, category or checker with the given id(s). You
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once).You can also use "--disable=all" to
|
||||
# disable everything first and then reenable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||
# --disable=W"
|
||||
disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating
|
||||
|
||||
|
||||
[REPORTS]
|
||||
|
||||
# Set the output format. Available formats are text, parseable, colorized, msvs
|
||||
# (visual studio) and html. You can also give a reporter class, eg
|
||||
# mypackage.mymodule.MyReporterClass.
|
||||
output-format=text
|
||||
|
||||
# Put messages in a separate file for each module / package specified on the
|
||||
# command line instead of printing them on stdout. Reports (if any) will be
|
||||
# written in a file name "pylint_global.[txt|html]".
|
||||
files-output=no
|
||||
|
||||
# Tells whether to display a full report or only the messages
|
||||
reports=yes
|
||||
|
||||
# Python expression which should return a note less than 10 (10 is the highest
|
||||
# note). You have access to the variables errors warning, statement which
|
||||
# respectively contain the number of errors / warnings messages and the total
|
||||
# number of statements analyzed. This is used by the global evaluation report
|
||||
# (RP0004).
|
||||
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
|
||||
|
||||
# Template used to display messages. This is a python new-style format string
|
||||
# used to format the message information. See doc for all details
|
||||
#msg-template=
|
||||
|
||||
|
||||
[VARIABLES]
|
||||
|
||||
# Tells whether we should check for unused import in __init__ files.
|
||||
init-import=no
|
||||
|
||||
# A regular expression matching the name of dummy variables (i.e. expectedly
|
||||
# not used).
|
||||
dummy-variables-rgx=_$|dummy
|
||||
|
||||
# List of additional names supposed to be defined in builtins. Remember that
|
||||
# you should avoid to define new builtins when possible.
|
||||
additional-builtins=
|
||||
|
||||
# List of strings which can identify a callback function by name. A callback
|
||||
# name must start or end with one of those strings.
|
||||
callbacks=cb_,_cb
|
||||
|
||||
|
||||
[LOGGING]
|
||||
|
||||
# Logging modules to check that the string format arguments are in logging
|
||||
# function parameter format
|
||||
logging-modules=logging
|
||||
|
||||
|
||||
[BASIC]
|
||||
|
||||
# List of builtins function names that should not be used, separated by a comma
|
||||
bad-functions=map,filter,input
|
||||
|
||||
# Good variable names which should always be accepted, separated by a comma
|
||||
good-names=i,j,k,ex,Run,_
|
||||
|
||||
# Bad variable names which should always be refused, separated by a comma
|
||||
bad-names=foo,bar,baz,toto,tutu,tata
|
||||
|
||||
# Colon-delimited sets of names that determine each other's naming style when
|
||||
# the name regexes allow several styles.
|
||||
name-group=
|
||||
|
||||
# Include a hint for the correct naming format with invalid-name
|
||||
include-naming-hint=no
|
||||
|
||||
# Regular expression matching correct function names
|
||||
function-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for function names
|
||||
function-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct variable names
|
||||
variable-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for variable names
|
||||
variable-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct constant names
|
||||
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Naming hint for constant names
|
||||
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||
|
||||
# Regular expression matching correct attribute names
|
||||
attr-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for attribute names
|
||||
attr-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct argument names
|
||||
argument-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for argument names
|
||||
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression matching correct class attribute names
|
||||
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Naming hint for class attribute names
|
||||
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||
|
||||
# Regular expression matching correct inline iteration names
|
||||
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Naming hint for inline iteration names
|
||||
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
|
||||
|
||||
# Regular expression matching correct class names
|
||||
class-rgx=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Naming hint for class names
|
||||
class-name-hint=[A-Z_][a-zA-Z0-9]+$
|
||||
|
||||
# Regular expression matching correct module names
|
||||
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Naming hint for module names
|
||||
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||
|
||||
# Regular expression matching correct method names
|
||||
method-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Naming hint for method names
|
||||
method-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||
|
||||
# Regular expression which should only match function or class names that do
|
||||
# not require a docstring.
|
||||
no-docstring-rgx=^_
|
||||
|
||||
# Minimum line length for functions/classes that require docstrings, shorter
|
||||
# ones are exempt.
|
||||
docstring-min-length=-1
|
||||
|
||||
|
||||
[ELIF]
|
||||
|
||||
# Maximum number of nested blocks for function / method body
|
||||
max-nested-blocks=5
|
||||
|
||||
|
||||
[SPELLING]
|
||||
|
||||
# Spelling dictionary name. Available dictionaries: none. To make it working
|
||||
# install python-enchant package.
|
||||
spelling-dict=
|
||||
|
||||
# List of comma separated words that should not be checked.
|
||||
spelling-ignore-words=
|
||||
|
||||
# A path to a file that contains private dictionary; one word per line.
|
||||
spelling-private-dict-file=
|
||||
|
||||
# Tells whether to store unknown words to indicated private dictionary in
|
||||
# --spelling-private-dict-file option instead of raising a message.
|
||||
spelling-store-unknown-words=no
|
||||
|
||||
|
||||
[FORMAT]
|
||||
|
||||
# Maximum number of characters on a single line.
|
||||
max-line-length=100
|
||||
|
||||
# Regexp for a line that is allowed to be longer than the limit.
|
||||
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
|
||||
|
||||
# Allow the body of an if to be on the same line as the test if there is no
|
||||
# else.
|
||||
single-line-if-stmt=no
|
||||
|
||||
# List of optional constructs for which whitespace checking is disabled. `dict-
|
||||
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
|
||||
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
|
||||
# `empty-line` allows space-only lines.
|
||||
no-space-check=trailing-comma,dict-separator
|
||||
|
||||
# Maximum number of lines in a module
|
||||
max-module-lines=1000
|
||||
|
||||
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
|
||||
# tab).
|
||||
indent-string=' '
|
||||
|
||||
# Number of spaces of indent required inside a hanging or continued line.
|
||||
indent-after-paren=4
|
||||
|
||||
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
|
||||
expected-line-ending-format=
|
||||
|
||||
|
||||
[MISCELLANEOUS]
|
||||
|
||||
# List of note tags to take in consideration, separated by a comma.
|
||||
notes=FIXME,XXX,TODO
|
||||
|
||||
|
||||
[SIMILARITIES]
|
||||
|
||||
# Minimum lines number of a similarity.
|
||||
min-similarity-lines=4
|
||||
|
||||
# Ignore comments when computing similarities.
|
||||
ignore-comments=yes
|
||||
|
||||
# Ignore docstrings when computing similarities.
|
||||
ignore-docstrings=yes
|
||||
|
||||
# Ignore imports when computing similarities.
|
||||
ignore-imports=no
|
||||
|
||||
|
||||
[TYPECHECK]
|
||||
|
||||
# Tells whether missing members accessed in mixin class should be ignored. A
|
||||
# mixin class is detected if its name ends with "mixin" (case insensitive).
|
||||
ignore-mixin-members=yes
|
||||
|
||||
# List of module names for which member attributes should not be checked
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=twisted.internet.reactor,leveldb
|
||||
|
||||
# List of classes names for which member attributes should not be checked
|
||||
# (useful for classes with attributes dynamically set). This supports can work
|
||||
# with qualified names.
|
||||
ignored-classes=twisted.internet,RequestMessage
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
# system, and so shouldn't trigger E1101 when accessed. Python regular
|
||||
# expressions are accepted.
|
||||
generated-members=lbrynet.lbrynet_daemon.LBRYDaemon.Parameters
|
||||
|
||||
|
||||
[IMPORTS]
|
||||
|
||||
# Deprecated modules which should not be used, separated by a comma
|
||||
deprecated-modules=regsub,TERMIOS,Bastion,rexec
|
||||
|
||||
# Create a graph of every (i.e. internal and external) dependencies in the
|
||||
# given file (report RP0402 must not be disabled)
|
||||
import-graph=
|
||||
|
||||
# Create a graph of external dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
ext-import-graph=
|
||||
|
||||
# Create a graph of internal dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
int-import-graph=
|
||||
|
||||
|
||||
[DESIGN]
|
||||
|
||||
# Maximum number of arguments for function / method
|
||||
max-args=5
|
||||
|
||||
# Argument names that match this expression will be ignored. Default to name
|
||||
# with leading underscore
|
||||
ignored-argument-names=_.*
|
||||
|
||||
# Maximum number of locals for function / method body
|
||||
max-locals=15
|
||||
|
||||
# Maximum number of return / yield for function / method body
|
||||
max-returns=6
|
||||
|
||||
# Maximum number of branch for function / method body
|
||||
max-branches=12
|
||||
|
||||
# Maximum number of statements in function / method body
|
||||
max-statements=50
|
||||
|
||||
# Maximum number of parents for a class (see R0901).
|
||||
max-parents=7
|
||||
|
||||
# Maximum number of attributes for a class (see R0902).
|
||||
max-attributes=7
|
||||
|
||||
# Minimum number of public methods for a class (see R0903).
|
||||
min-public-methods=2
|
||||
|
||||
# Maximum number of public methods for a class (see R0904).
|
||||
max-public-methods=20
|
||||
|
||||
# Maximum number of boolean expressions in a if statement
|
||||
max-bool-expr=5
|
||||
|
||||
|
||||
[CLASSES]
|
||||
|
||||
# List of method names used to declare (i.e. assign) instance attributes.
|
||||
defining-attr-methods=__init__,__new__,setUp
|
||||
|
||||
# List of valid names for the first argument in a class method.
|
||||
valid-classmethod-first-arg=cls
|
||||
|
||||
# List of valid names for the first argument in a metaclass class method.
|
||||
valid-metaclass-classmethod-first-arg=mcs
|
||||
|
||||
# List of member names, which should be excluded from the protected access
|
||||
# warning.
|
||||
exclude-protected=_asdict,_fields,_replace,_source,_make
|
||||
|
||||
|
||||
[EXCEPTIONS]
|
||||
|
||||
# Exceptions that will emit a warning when being caught. Defaults to
|
||||
# "Exception"
|
||||
overgeneral-exceptions=Exception
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
matrix:
|
||||
include:
|
||||
- os: linux
|
||||
sudo: required
|
||||
dist: trusty
|
||||
# dh-virtualenv requires that we use the same python interpreter
|
||||
# that comes with the system, so we don't want to use anything that
|
||||
# travis would try to set-up for us in python
|
||||
language: generic
|
||||
- os: osx
|
||||
# Use generic language for osx
|
||||
# python 2.7 is broken on osx on travis, so follow we have to specify the installation ourselves
|
||||
# https://github.com/travis-ci/travis-ci/issues/2312#issuecomment-195620855
|
||||
language: generic
|
||||
osx_image: xcode7.3
|
||||
|
||||
notifications:
|
||||
slack:
|
||||
secure: "Am13HPtpgCMljh0MDVuoFHvQXB8yhf4Kvf/qAeSp5N0vsHGL70CSF9Ahccw8dVPE6mbuak1OGtSUb6/UaErLHkpz3ztaRLkDa9x7CmBB3Kynnh8oO2VbB7b/2ROULqkhF4VZmAnNfwrQrbC3gs8Sybp261Nyc7y4ww15xDYBrk2fyq4ds2DCaJdRxfJUJFonrZ6KXr3fVaXosO6cjuyS8eRodcmrqsT4cCtinjNTD1hGWoH107E4ObSmpVelxQO193KhNJMRiLlEcVkvYUOqIWBtwdGHbNE/6Yeuq1TXgKJ0KeJWAmW3wTfUYNngGXNAsyCnrhul5TKNevNzfIAQZHvRsczYiWPJV6LtohHT0CcUiCXJtvEPOyahEBfwK3etY/xxFqny7N9OEmpdW2sgsEPNPX2LJynJti2rQA9SuAD1ogR3ZpDy/NXoaAZf8PTdPcuNUMULV9PGG7tVrLBecO/W1qO6hdFxwlLdgqGLxAENZgGp++v/DhPk/WvtmHj7iTbRq0nxaTWyX5uKOn2ADH+k/yfutjv6BsQU9xNyPeZEEtuEpc6X6waiYn/8G9vl9PecvKC5H0MgsZ6asAxmg7mZ3VSMFG7mo8ENeOhSZ0Oz6ZTBILL3wFccZA9uJIq7NWmqC9dRiGiuKXBB62No7sINoHg3114e2xYa9qvNmGg="
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
- $HOME/Library/Caches/pip
|
||||
- $TRAVIS_BUILD_DIR/cache/wheel
|
||||
|
||||
before_install:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/travis/setup_osx.sh; fi
|
||||
|
||||
install:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./packaging/travis/install_dependencies_and_run_tests.sh; fi
|
||||
|
||||
before_script:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then openssl aes-256-cbc -k "$ENCRYPTION_SECRET" -in packaging/osx/certs/dist.cer.enc -d -a -out packaging/osx/certs/dist.cer; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then openssl aes-256-cbc -k "$ENCRYPTION_SECRET" -in packaging/osx/certs/dist.p12.enc -d -a -out packaging/osx/certs/dist.p12; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./packaging/osx/add-key.sh; fi
|
||||
|
||||
script:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then bash packaging/ubuntu/ubuntu_package_setup.sh -t; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then cd packaging/osx/lbry-osx-app && ./setup_app.sh && cd $TRAVIS_BUILD_DIR; fi
|
||||
# fail the build if this is a build for a tag and we don't have the versions matching; allow tags that start with 'test' to pass
|
||||
- if [[ -n "${TRAVIS_TAG}" ]]; then if [[ "${TRAVIS_TAG}" == test* ]] || [[ "v`python setup.py -V`" = "${TRAVIS_TAG}" ]]; then true; else false; fi; fi
|
||||
|
||||
deploy:
|
||||
- provider: releases
|
||||
file: "${TRAVIS_BUILD_DIR}/`python setup.py --name`_`python setup.py -V`_amd64.deb"
|
||||
skip_cleanup: true
|
||||
prerelease: true
|
||||
on:
|
||||
tags: true
|
||||
condition: "$TRAVIS_OS_NAME = linux"
|
||||
# this is the oauth token for the lbry-ci user
|
||||
api_key:
|
||||
secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc=
|
||||
- provider: releases
|
||||
file: "${TRAVIS_BUILD_DIR}/packaging/osx/lbry-osx-app/`python setup.py --name`.`python setup.py -V`.dmg"
|
||||
skip_cleanup: true
|
||||
prerelease: true
|
||||
on:
|
||||
tags: true
|
||||
condition: "$TRAVIS_OS_NAME = osx"
|
||||
# this is the oauth token for the lbry-ci user
|
||||
api_key:
|
||||
secure: nKdWGROnLNodx9k9nWvq2wezkPvSVL8zF63qjPXjhdOe9kCUbcLp7WnFDYpn9EJj4Pofq/ejeCHwjA+5x7JUP3Szk7SlJV61B4c/5hl64rl7oSKOoKskjdE2jaOG3CJuOUrh0yQ59U3vMMABcsnw/wJaCIuu/erdPIm8g8R+stu1YHOGtl5Y9WiW+zLJn2vc3GooV1TWtki9EnrmFfw0Vrqc4RMVMFB1ojE7ggrK1LIwcmGSbLIYzker1ZRz8SCy+84sGk4//V+2i2NNiz5AkPuG7BBGrU2twE9nD23IlruJAdVdi71P3ytAmi0kKyvxIU4VeNaqyTk9zeL5IB9J5IIgvekHgKcsKhFUZ6QcXT1Xfxl4ELftvWCTHWiewnXFdqLcG9GZiUaE6+7wdalwDAP3tqS2emiibetlBZERHR+RMR00ej+1MBYWGMlTse/0Tglndv0a2qqgAJYLKPRT02hTRYGxZ1MrJe+WGnChRmzwgLVTIgZuiDciFOahN0TYGSORk6OpnZBsxvpzSqDw5UDJx0BmbJ1xMNDFbOs8ubZ9yIpB9yNMGw66FPacOF61XNYnmA68ILC28UtOFKuuHLrUPbM5JmQkDVhtTfFbBnyHefyCLAL4MHvJJKGi1oaOXjYaJ/J095h636/kQ0cHHuVMgoWUQZOQ44xRAz7tMuc=
|
||||
|
||||
env:
|
||||
global:
|
||||
# needed to unlock the identity file when its on the keychain
|
||||
- secure: "aqgVNHfeh6JUqTTWG5+W+tTsIJkePS6HyLkcZlq6ODYrfdGKa90EeV4q4wdGfP5IkrBbf+WBFACDGVp574E7vfMLMNKUDuUtgnGVPwqOIhjG82ud8Xa7qF7lsw50QOnRYYVd2GLlCIzk8sffT5ncjSPN2ClNVM5iTwCkC8TNSAVnEJxu6bG6hcaT4uCWWjs0m+39O1xJwxJ4vTjpE/gy+j2FTSaUR5cNavOVCyJqpeKlga9aoBVsQHWvxlYurdWbwRLwVIV7bDE+sYPnwn1nMFQpx5RZ1AX1Z2UxNFKcYzLJgcWe85OjxLyT4udX+XZ9SLsdOjm1n201OLEKsmTxmHS8yqpbu6+pKQ1rLMEVpgPGfS8DdtqZyb3z0u6q4jztpm+uBe8hnFgwQFGXO63nOQsI0n1PMR3evAnVx7jYt7y/UALs2A9yjosMwDqgql1FMhyd0OJGo8Ky8YpDsr2J5zGVvIqt2/N+lP+SOe2D1J/5EhGlY0o4tqIAFskh3q8/GR6UGm70KT2l4LJqrU1WGrGxPJ+HoOEmvG6eqLesk03fAX3v5+DgXZWErnzXMIOGDPaFVpmx+G9VZTIlmf+3Wbu9TnZE5PRwFTdP1rqjGpjUhHeF0VTc2qgNq1OfL98CBO1wLgA195+em58cELalnwDMWUTmY1Jt1kUuAtCc11U="
|
||||
# used to decypt the aes decrypt the certificate and identity files
|
||||
- secure: "FzftiEBFMngQIci5KZ+tpKs/BubalhWXJ9f8yogGNVa3XplLheUXYgcsNU8sYT2MJaKAHN9fSjHWb67UtKT7yXddXxeOPnW6wJ3ua/FXnpynsmeGKTfh3stvgjpzdXous67uHmCWMMklfb6z7UuDohjUMAe5n5HKw5tq1RzTKpc2kJacOC6qUT5laUOvULyCaO9E9HmbHeR5ZeXAC6pnzX2ccsSrcXvPozHzBIZ9RyothKs+CZw8PEuJo07RRL8meboegqYOUrOYuz6A2gS2mZJoy59ivZKOFxS5shEuv2Jt80RyfyxBoUpKFq8OG3Am5nAEzDiTIGzmIoKDEGKeTagk/sEtSZXiMDkzDT4GX/j1rUNLCBU87bXEFS2zfRsrfg8c1XZPIzDYBT1PY2QtLBdddF5zzDoKdPLJ3sjN+fZFE5RlnwfwnMHriVRZZlzjcdk0Z06gKTBCxUg5BZamnOOK+K8qunMJXVS+Vmi5u4RoTZZiCosUlYKnSKJ8suO9C0+znxoViusPqP4ONprNHgDoZw+UKio84QW3PrZv9h4zH/D+msDgJRZ0ceqDD+6Wz1J8Mm5ptW1GOLh/IU12TPXjxteqh0Um2vv5eIPmjK9uEK666kK7PqtPDkYhAfWvF+nmOOyPMJfbP4MW/i9WHNF4ghsIMbPKfqNhgSmfrYw="
|
||||
@@ -1,3 +0,0 @@
|
||||
# Change Log
|
||||
|
||||
This changelog is no longer up to date. For release notes, see https://github.com/lbryio/lbry-sdk/releases.
|
||||
@@ -1,3 +0,0 @@
|
||||
## Contributing to LBRY
|
||||
|
||||
https://lbry.tech/contribute
|
||||
@@ -0,0 +1,85 @@
|
||||
#### Getting LBRY for development
|
||||
|
||||
Q: How do I get lbry for command line?
|
||||
|
||||
A: In order to run lbry from command line, you need more than the packaged app/deb.
|
||||
|
||||
######On OS X
|
||||
|
||||
You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio/lbry-setup/master/lbry_setup_osx.sh | sudo bash` in a terminal. This script will install lbrynet and its dependancies, as well as the app.
|
||||
|
||||
######On Linux
|
||||
|
||||
On Ubuntu or Mint you can install the prerequisites and lbrynet by running
|
||||
|
||||
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
sudo python setup.py install
|
||||
|
||||
#### Using LBRY
|
||||
|
||||
Q: How do I run lbry from command line?
|
||||
|
||||
A: The command is `lbrynet-daemon`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I stop lbry from the command line?
|
||||
|
||||
A: You can ctrl-c or run `stop-lbrynet-daemon`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I run lbry with lbrycrdd (the blockchain node application)?
|
||||
|
||||
A: Start lbry with the --wallet flag set: `lbrynet-daemon --wallet=lbrycrd`
|
||||
|
||||
Note: when you change the wallet it is persistant until you specify you want to use another wallet - lbryum - with the --wallet flag again.
|
||||
|
||||
***********
|
||||
|
||||
Q: Where are all the behind the scenes files?
|
||||
|
||||
A: On linux, the relevant directories are `~/.lbrynet`, `~/.lbrycrd`, and `~/.lbryum`, depending on which wallets you've used. On OS X, the folders of interest are `~/Library/Application Support/LBRY`, `~/.lbrycrd` and `~/.lbryum`, also depending on which wallets you've used.
|
||||
|
||||
***********
|
||||
|
||||
Q: How can I see the log in the console?
|
||||
|
||||
A: Run lbry with the --log-to-console flag set: `lbrynet-daemon --log-to-console`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I specify a web-UI to use?
|
||||
|
||||
A: If the files for the UI you'd like to use are storred locally on your computer, start lbry with the --ui flag: `lbrynet-daemon --ui=/full/path/to/ui/files/root/folder`
|
||||
|
||||
Note, once set with the UI flag the given UI will be cached by lbry and used as the default going forward. Also, it will only successfully load a UI if it contains a conforming requirements.txt file to specify required lbrynet and lbryum versions. [Here](https://github.com/lbryio/lbry-web-ui/blob/master/dist/requirements.txt) is an example requirements.txt file.
|
||||
|
||||
To reset your ui to pull from lbryio, or to try a UI still in development, run lbry with the --branch flag: `lbrynet=daemon --branch=master`
|
||||
|
||||
***********
|
||||
|
||||
Q: How do I see the list of API functions I can call, and how do I call them?
|
||||
|
||||
A: Here is an example script to get the documentation for the various API calls. To use any of the functions displayed, just provide any specified arguments in a dictionary.
|
||||
|
||||
Note: the lbry api can only be used while either the app or lbrynet-daemon command line are running
|
||||
|
||||
import sys
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
try:
|
||||
from lbrynet.conf import API_CONNECTION_STRING
|
||||
except:
|
||||
print "You don't have lbrynet installed!"
|
||||
sys.exit(0)
|
||||
|
||||
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
if not api.is_running():
|
||||
print api.daemon_status()
|
||||
else:
|
||||
for func in api.help():
|
||||
print "%s:\n%s" % (func, api.help({'function': func}))
|
||||
|
||||
+45
-86
@@ -1,108 +1,67 @@
|
||||
# Installing LBRY
|
||||
#### Installing the LBRY app
|
||||
--------------------------
|
||||
|
||||
If only the JSON-RPC API server is needed, the recommended way to install LBRY is to use a pre-built binary. We provide binaries for all major operating systems. See the [README](README.md)!
|
||||
Installing LBRY is simple. You can get a dmg installer for OS X (Mavericks and up) or a .deb for linux [here](https://lbry.io/get).
|
||||
|
||||
These instructions are for installing LBRY from source, which is recommended if you are interested in doing development work or LBRY is not available on your operating system (godspeed, TempleOS users).
|
||||
##### OS X
|
||||
Just drag and drop LBRY.app into your applications folder (replacing any older versions). When it's running you'll have a LBRY icon in your status bar.
|
||||
|
||||
Here's a video walkthrough of this setup, which is itself hosted by the LBRY network and provided via [spee.ch](https://github.com/lbryio/spee.ch):
|
||||
[](https://spee.ch/967f99344308f1e90f0620d91b6c93e4dfb240e0/lbrynet-dev-setup.mp4)
|
||||
##### Linux
|
||||
Double click the .deb file and follow the prompts. The app can be started by searching "LBRY", and it can be turned off by clicking the red 'x' in the browser interface.
|
||||
|
||||
## Prerequisites
|
||||
On both systems you can also open the UI while the app is running by going to lbry://lbry in Firefox or Safari, or localhost:5279 in Chrome.
|
||||
|
||||
Running `lbrynet` from source requires Python 3.7 or higher. Get the installer for your OS [here](https://www.python.org/downloads/release/python-370/).
|
||||
|
||||
After installing python 3, you'll need to install some additional libraries depending on your operating system.
|
||||
|
||||
### macOS
|
||||
#### Installing LBRY command line
|
||||
--------------------------
|
||||
|
||||
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
|
||||
##### OS X
|
||||
You can install LBRY command line by running `curl -sL https://rawgit.com/lbryio/lbry-setup/master/lbry_setup_osx.sh | sudo bash` in a terminal. This script will install lbrynet and its dependancies, as well as the app. You can start LBRY by either starting the app or by running `lbrynet-daemon` from a terminal.
|
||||
|
||||
These environment variables also need to be set:
|
||||
1. PYTHONUNBUFFERED=1
|
||||
2. EVENT_NOKQUEUE=1
|
||||
|
||||
Remaining dependencies can then be installed by running:
|
||||
|
||||
```
|
||||
brew install python protobuf
|
||||
```
|
||||
|
||||
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/.
|
||||
|
||||
### Linux
|
||||
|
||||
On Ubuntu (16.04 minimum, we recommend 18.04), install the following:
|
||||
|
||||
```
|
||||
sudo add-apt-repository ppa:deadsnakes/ppa
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential python3.7 python3.7-dev git python3.7-venv libssl-dev python-protobuf
|
||||
```
|
||||
|
||||
On Raspbian, you will also need to install `python-pyparsing`.
|
||||
|
||||
If you're running another Linux distro, install the equivalent of the above packages for your system.
|
||||
|
||||
## Installation
|
||||
|
||||
### Linux/Mac
|
||||
|
||||
To install on Linux/Mac:
|
||||
|
||||
```
|
||||
Clone the repository:
|
||||
$ git clone https://github.com/lbryio/lbry-sdk.git
|
||||
$ cd lbry-sdk
|
||||
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
$ python3.7 -m venv lbry-venv
|
||||
##### Linux
|
||||
On Ubuntu or Mint you can install the prerequisites and lbrynet by running
|
||||
|
||||
Activating lbry-sdk virtual environment:
|
||||
$ source lbry-venv/bin/activate
|
||||
|
||||
Make sure you're on Python 3.7+ (as the default Python in virtual environment):
|
||||
$ python --version
|
||||
|
||||
Install packages:
|
||||
$ make install
|
||||
|
||||
If you are on Linux and using PyCharm, generates initial configs:
|
||||
$ make idea
|
||||
```
|
||||
sudo apt-get install libgmp3-dev build-essential python2.7 python2.7-dev python-pip git
|
||||
git clone https://github.com/lbryio/lbry.git
|
||||
cd lbry
|
||||
sudo python setup.py install
|
||||
```
|
||||
|
||||
To verify your installation, `which lbrynet` should return a path inside of the `lbry-venv` folder created by the `python3.7 -m venv lbry-venv` command.
|
||||
To start LBRY, run `lbrynet-daemon` in a terminal.
|
||||
|
||||
### Windows
|
||||
#### On windows:
|
||||
|
||||
To install on Windows:
|
||||
Install [mingw32](http://www.mingw.org/) base and c++ compiler.
|
||||
|
||||
```
|
||||
Clone the repository:
|
||||
> git clone https://github.com/lbryio/lbry-sdk.git
|
||||
> cd lbry-sdk
|
||||
Add C:\MinGW\bin to the windows PATH.
|
||||
|
||||
Create a Python virtual environment for lbry-sdk:
|
||||
> python -m venv lbry-venv
|
||||
Enable distutils to compile with mingw32 by creating a distutils.cfg file in *PYTHONPATH\Lib\distutils* containing
|
||||
```
|
||||
[build]
|
||||
compiler = mingw32
|
||||
```
|
||||
|
||||
Activating lbry-sdk virtual environment:
|
||||
> lbry-venv\Scripts\activate
|
||||
If using virtualenv, copy the *PYTHONPATH\Lib\distutils* directory to the virtualenv.
|
||||
|
||||
Install packages:
|
||||
> pip install -e .
|
||||
```
|
||||
It's recommended to use [Unofficial Windows Binaries for Python Extension Packages](http://www.lfd.uci.edu/~gohlke/pythonlibs/) for as many of the required packages as possible.
|
||||
Currently, available binaries include:
|
||||
- Cython
|
||||
- Twisted
|
||||
- Zope.interface
|
||||
- pywin32
|
||||
- Yapsy
|
||||
- cx_Freeze
|
||||
- requests
|
||||
- gmpy
|
||||
|
||||
## Run the tests
|
||||
Install the each of the preceding binaries with `pip install *.whl`
|
||||
|
||||
To run the unit tests from the repo directory:
|
||||
Install pywin32 system files by run `python.exe Scripts\pywin32_postinstall.py -install` from an elevated command prompt.
|
||||
|
||||
```
|
||||
python -m unittest discover tests.unit
|
||||
```
|
||||
Install the rest of the required modules with the standard `pip install module` command
|
||||
|
||||
## Usage
|
||||
|
||||
To start the API server:
|
||||
`lbrynet start`
|
||||
|
||||
|
||||
Happy hacking!
|
||||
The one module which requires additional work is [miniupnpc](https://pypi.python.org/pypi/miniupnpc/1.9).
|
||||
Download the source and compile with MinGW by running `mingw32make.bat`
|
||||
Then install the module by running `python setupmingw32.py install`
|
||||
@@ -1,15 +1,22 @@
|
||||
The MIT License (MIT)
|
||||
Copyright (c) 2015, LBRY, Inc.
|
||||
All rights reserved.
|
||||
|
||||
Copyright (c) 2015-2020 LBRY Inc
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+2
-4
@@ -1,4 +1,2 @@
|
||||
include README.md
|
||||
include CHANGELOG.md
|
||||
include LICENSE
|
||||
recursive-include lbry *.txt *.py
|
||||
include lbrynet/lbrynet_console/plugins/blindrepeater.yapsy-plugin
|
||||
recursive-include lbrynet/lbrynet_gui *.gif *.ico *.xbm *.conf
|
||||
@@ -1,20 +0,0 @@
|
||||
.PHONY: tools lint test idea
|
||||
|
||||
lint:
|
||||
pylint --rcfile=setup.cfg lbry
|
||||
#mypy --ignore-missing-imports lbry
|
||||
|
||||
test:
|
||||
tox
|
||||
|
||||
idea:
|
||||
mkdir -p .idea
|
||||
cp -r scripts/idea/* .idea
|
||||
|
||||
start:
|
||||
dropdb lbry --if-exists
|
||||
createdb lbry
|
||||
lbrynet start node \
|
||||
--db-url=postgresql:///lbry --workers=0 --console=advanced --no-spv-address-filters \
|
||||
--lbrycrd-rpc-user=lbry --lbrycrd-rpc-pass=somethingelse \
|
||||
--lbrycrd-dir=${HOME}/.lbrycrd --data-dir=/tmp/tmp-lbrynet
|
||||
@@ -1,60 +1,72 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry-sdk/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://ci.lbry.tech/lbry/lbry-sdk)
|
||||
[](https://travis-ci.org/lbryio/lbry)
|
||||
|
||||
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
# LBRYnet
|
||||
|
||||
LBRY SDK for Python is currently the most fully featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components include:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbry.dht](https://github.com/lbryio/lbry-sdk/tree/master/lbry/dht)).
|
||||
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbry.blob_exchange](https://github.com/lbryio/lbry-sdk/tree/master/lbry/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbry.schema](https://github.com/lbryio/lbry-sdk/tree/master/lbry/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbry.wallet](https://github.com/lbryio/lbry-sdk/tree/master/lbry/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbry.extras.daemon](https://github.com/lbryio/lbry-sdk/tree/master/lbry/extras/daemon)).
|
||||
LBRYnet is a fully decentralized network for distributing data. It consists of peers uploading
|
||||
and downloading data from other peers, possibly in exchange for payments, and a distributed hash
|
||||
table, used by peers to discover other peers.
|
||||
|
||||
## Installation
|
||||
|
||||
Our [releases page](https://github.com/lbryio/lbry-sdk/releases) contains pre-built binaries of the latest release, pre-releases, and past releases for macOS, Debian-based Linux, and Windows. [Automated travis builds](http://build.lbry.io/daemon/) are also available for testing.
|
||||
Download the [latest release](https://github.com/lbryio/lbry/releases/latest) or see [INSTALL.md](INSTALL.md) for manual installation.
|
||||
|
||||
## Usage
|
||||
## Overview
|
||||
|
||||
Run `lbrynet start` to launch the API server.
|
||||
On LBRYnet, data is broken into chunks, and each chunk is specified by its sha384 hash sum. This
|
||||
guarantees that peers can verify the correctness of each chunk without having to know anything
|
||||
about its contents, and can confidently re-transmit the chunk to other peers. Peers wishing to
|
||||
transmit chunks to other peers announce to the distributed hash table that they are associated
|
||||
with the sha384 hash sum in question. When a peer wants to download that chunk from the network,
|
||||
it asks the distributed hash table which peers are associated with that sha384 hash sum. The
|
||||
distributed hash table can also be used more generally. It simply stores IP addresses and
|
||||
ports which are associated with 384-bit numbers, and can be used by any type of application to
|
||||
help peers find each other. For example, an application for which clients don't know all of the
|
||||
necessary chunks may use some identifier, chosen by the application, to find clients which do
|
||||
know all of the necessary chunks.
|
||||
|
||||
By default, `lbrynet` will provide a JSON-RPC server at `http://localhost:5279`. It is easy to interact with via cURL or sane programming languages.
|
||||
## For Developers
|
||||
|
||||
Our [quickstart guide](https://lbry.tech/playground) provides a simple walkthrough and examples for learning.
|
||||
The bundled LBRY application uses the lbrynet JSONRPC api found in `lbrynet.lbrynet_daemon.LBRYDaemon`. This api allows for applications and web services like the lbry browser UI to interact with lbrynet. If you've installed lbrynet, you can run `lbrynet-daemon` without running the app. While the app or `lbrynet-daemon` is running, you can use the following to show the help for all the available commands:
|
||||
|
||||
With the daemon running, `lbrynet commands` will show you a list of commands.
|
||||
```
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
The full API is documented [here](https://lbry.tech/api/sdk).
|
||||
try:
|
||||
from lbrynet.conf import API_CONNECTION_STRING
|
||||
except:
|
||||
print "You don't have lbrynet installed!"
|
||||
API_CONNECTION_STRING = "http://localhost:5279/lbryapi"
|
||||
|
||||
api = JSONRPCProxy.from_url(API_CONNECTION_STRING)
|
||||
if not api.is_running():
|
||||
print api.daemon_status()
|
||||
else:
|
||||
for func in api.help():
|
||||
print "%s:\n%s" % (func, api.help({'function': func}))
|
||||
```
|
||||
|
||||
## Recommended hardware
|
||||
If you've installed lbrynet, it comes with a file sharing application, called `lbrynet-daemon`, which breaks
|
||||
files into chunks, encrypts them with a symmetric key, computes their sha384 hash sum, generates
|
||||
a special file called a 'stream descriptor' containing the hash sums and some other file metadata,
|
||||
and makes the chunks available for download by other peers. A peer wishing to download the file
|
||||
must first obtain the 'stream descriptor' and then may open it with his `lbrynet-daemon` client,
|
||||
download all of the chunks by locating peers with the chunks via the DHT, and then combine the
|
||||
chunks into the original file, according to the metadata included in the 'stream descriptor'.
|
||||
|
||||
The minimum hardware for a full node is 16cpus, 92gb of RAM, and 160gb of NVMe storage. The recommended hardware is 32cpus, 128gb of RAM, and 160gb of NVMe storage.
|
||||
For detailed instructions, see [INSTALL.md](INSTALL.md) and [RUNNING.md](RUNNING.md).
|
||||
|
||||
## Running from source
|
||||
Documentation: doc.lbry.io (may be out of date)
|
||||
|
||||
Installing from source is also relatively painless. Full instructions are in [INSTALL.md](INSTALL.md)
|
||||
Source code: https://github.com/lbryio/lbry
|
||||
|
||||
## Contributing
|
||||
To contribute, [join us on Slack](https://lbry-slackin.herokuapp.com/) or contact josh@lbry.io. Pull requests are also welcome.
|
||||
|
||||
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
|
||||
## Support
|
||||
|
||||
Please open an issue and describe your situation in detail. We will respond as soon as we can.
|
||||
|
||||
For private issues, contact josh@lbry.io.
|
||||
|
||||
## License
|
||||
|
||||
This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
The documentation for the API can be found [here](https://lbry.tech/api/sdk).
|
||||
|
||||
Daemon defaults, ports, and other settings are documented [here](https://lbry.tech/resources/daemon-settings).
|
||||
|
||||
Settings can be configured using a daemon-settings.yml file. An example can be found [here](https://github.com/lbryio/lbry-sdk/blob/master/example_daemon_settings.yml).
|
||||
See [LICENSE](LICENSE)
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
How to watch It's a Wonderful Life via LBRY
|
||||
|
||||
## Quickest quick guide
|
||||
|
||||
Create a directory called lbry, and go into that directory
|
||||
|
||||
Download the file https://raw.githubusercontent.com/lbryio/lbry-setup/master/lbry_setup.sh and run it in that directory
|
||||
|
||||
Once it's done building, type:
|
||||
|
||||
```
|
||||
lbrynet-console
|
||||
```
|
||||
|
||||
A console application will load, and after a moment you will be presented with a ">" signifying
|
||||
that the console is ready for commands.
|
||||
|
||||
If it's your first time running lbrynet-console, you should be given some credits to test the
|
||||
network. If they don't show up within a minute or two, let us know, and we'll send you some.
|
||||
|
||||
After your credits have shown up, type
|
||||
|
||||
```
|
||||
get wonderfullife
|
||||
```
|
||||
|
||||
into the prompt and hit enter.
|
||||
|
||||
You will be asked if you want to cancel, change options, save, and perhaps stream the file.
|
||||
|
||||
Enter 's' for save and then hit enter.
|
||||
|
||||
The file should start downloading. Enter the command 'status' to check on the status of files
|
||||
that are in the process of being downloaded.
|
||||
|
||||
To stop lbrynet-console, enter the command 'exit'.
|
||||
|
||||
|
||||
## Slightly longer install guide
|
||||
|
||||
### Installing lbrycrd, the full blockchain client
|
||||
|
||||
Note: this process takes upwards of an hour and is not necessary to use lbrynet.
|
||||
|
||||
```
|
||||
sudo apt-get install build-essential libtool autotools-dev autoconf pkg-config libssl-dev libboost-all-dev libdb-dev libdb++-dev libqt4-dev libprotobuf-dev protobuf-compiler git
|
||||
git clone --depth=1 -b alpha https://github.com/lbryio/lbrycrd.git
|
||||
cd lbrycrd
|
||||
./autogen.sh
|
||||
./configure --with-incompatible-bdb --without-gui
|
||||
|
||||
make
|
||||
```
|
||||
|
||||
When make has completed, create the directory where LBRYcrd data will be stored. ~/.lbrycrd is where LBRYcrd will look by default and so is recommended.
|
||||
|
||||
```
|
||||
mkdir ~/.lbrycrd
|
||||
echo 'rpcuser=rpcuser
|
||||
rpcpassword=rpcpassword' > ~/.lbrycrd/lbrycrd.conf
|
||||
# (use a long random password if your computer is on a network anyone else has access to. e.g, pwgen -s 20)
|
||||
cd ..
|
||||
|
||||
```
|
||||
|
||||
### Installing lbrynet from source
|
||||
|
||||
Acquire the LBRYnet source code from https://github.com/lbryio/lbry
|
||||
|
||||
```
|
||||
cd lbry
|
||||
sudo apt-get install libgmp3-dev build-essential python-dev python-pip
|
||||
```
|
||||
|
||||
(with virtualenv)
|
||||
|
||||
```
|
||||
python-virtualenv
|
||||
|
||||
virtualenv .
|
||||
|
||||
source bin/activate
|
||||
|
||||
python setup.py install
|
||||
|
||||
```
|
||||
|
||||
to deactivate the virtualenv later:
|
||||
|
||||
```
|
||||
deactivate
|
||||
```
|
||||
|
||||
to reactivate it, go to the directory in which you created it and:
|
||||
|
||||
```
|
||||
source bin/activate
|
||||
```
|
||||
|
||||
(without virtualenv)
|
||||
|
||||
```
|
||||
python setup.py build bdist_egg
|
||||
|
||||
sudo python setup.py install
|
||||
```
|
||||
|
||||
## Slightly longer running guide
|
||||
|
||||
### lbrynet-console can be set to use lbrycrdd instead of the built in lightweight client.
|
||||
|
||||
To run lbrynet-console with lbrycrdd:
|
||||
|
||||
```
|
||||
lbrynet-console </path/to/lbrycrdd>
|
||||
```
|
||||
|
||||
If lbrycrdd is not already running, lbrynet will launch it at that path, and will shut it down
|
||||
when lbrynet exits. If lbrycrdd is already running, lbrynet will not launch it and will not
|
||||
shut it down, but it will connect to it and use it.
|
||||
|
||||
### Running lbrycrdd manually
|
||||
|
||||
From the lbrycrd directory, run:
|
||||
|
||||
```
|
||||
./src/lbrycrdd -server -daemon
|
||||
```
|
||||
|
||||
If you want to mine LBC, also use the flag '-gen', so:
|
||||
|
||||
```
|
||||
./src/lbrycrdd -server -daemon -gen
|
||||
```
|
||||
|
||||
Warning: This will put a heavy load on your CPU
|
||||
|
||||
It will take a few minutes for your client to download the whole block chain.
|
||||
|
||||
To shut lbrycrdd down: from the lbrycrd directory, run:
|
||||
|
||||
```
|
||||
./src/lbrycrd-cli stop
|
||||
```
|
||||
|
||||
Any questions or problems, email jimmy@lbry.io
|
||||
@@ -1,5 +0,0 @@
|
||||
FROM ubuntu:20.04
|
||||
COPY lbrynet /bin
|
||||
RUN lbrynet --version
|
||||
ENTRYPOINT ["lbrynet"]
|
||||
CMD ["start", "node"]
|
||||
@@ -1,8 +0,0 @@
|
||||
FROM ubuntu:20.04
|
||||
RUN apt-get update && \
|
||||
apt-get install -y wget unzip && \
|
||||
wget -nv https://build.lbry.io/lbrycrd/block_info_fix_try2/lbrycrd-linux.zip && \
|
||||
unzip -d /bin lbrycrd-linux.zip && \
|
||||
rm -rf lbrycrd-linux.zip /var/lib/apt/lists/*
|
||||
RUN lbrycrdd --version
|
||||
ENTRYPOINT ["lbrycrdd"]
|
||||
@@ -1,41 +0,0 @@
|
||||
version: "3.8"
|
||||
volumes:
|
||||
lbrycrd-data:
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:12
|
||||
environment:
|
||||
POSTGRES_USER: lbry
|
||||
POSTGRES_PASSWORD: lbry
|
||||
lbrycrd:
|
||||
image: lbry/lbrycrd
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.lbrycrd
|
||||
volumes:
|
||||
- lbrycrd-data:/root/.lbrycrd
|
||||
command: >
|
||||
-rpcbind=lbrycrd
|
||||
-rpcallowip=0.0.0.0/0
|
||||
-rpcuser=lbryuser
|
||||
-rpcpassword=lbrypass
|
||||
-zmqpubhashblock=tcp://lbrycrd:29000
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:fast_wallet_server_sync
|
||||
depends_on:
|
||||
- postgres
|
||||
- lbrycrd
|
||||
volumes:
|
||||
- lbrycrd-data:/lbrycrd
|
||||
command: >
|
||||
start
|
||||
--full-node
|
||||
--api=0.0.0.0:5279
|
||||
--db-url=postgresql://lbry:lbry@postgres:5432/lbry
|
||||
--workers=12
|
||||
--console=basic
|
||||
--no-spv-address-filters
|
||||
--lbrycrd-rpc-host=lbrycrd
|
||||
--lbrycrd-rpc-user=lbryuser
|
||||
--lbrycrd-rpc-pass=lbrypass
|
||||
--lbrycrd-dir=/lbrycrd
|
||||
@@ -1,42 +0,0 @@
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
import lbry.build_info as build_info_mod
|
||||
|
||||
log = logging.getLogger()
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
def _check_and_set(d: dict, key: str, value: str):
|
||||
try:
|
||||
d[key]
|
||||
except KeyError:
|
||||
raise Exception(f"{key} var does not exist in {build_info_mod.__file__}")
|
||||
d[key] = value
|
||||
|
||||
|
||||
def main():
|
||||
build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
|
||||
|
||||
commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
|
||||
if commit_hash is None:
|
||||
raise ValueError("Commit hash not found in env vars")
|
||||
_check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
|
||||
|
||||
docker_tag = os.getenv('DOCKER_TAG')
|
||||
if docker_tag:
|
||||
_check_and_set(build_info, "DOCKER_TAG", docker_tag)
|
||||
_check_and_set(build_info, "BUILD", "docker")
|
||||
else:
|
||||
ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
|
||||
_check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
|
||||
|
||||
log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
|
||||
with open(build_info_mod.__file__, 'w') as f:
|
||||
f.write("\n".join([f"{k} = \"{v}\"" for k, v in build_info.items()]) + "\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
-4985
File diff suppressed because it is too large.
Load diff
@@ -1,25 +0,0 @@
|
||||
# This is an example daemon_settings.yml file.
|
||||
# See https://lbry.tech/resources/daemon-settings for all configuration keys and values
|
||||
|
||||
share_usage_data: True
|
||||
|
||||
lbryum_servers:
|
||||
- lbryumx1.lbry.com:50001
|
||||
- lbryumx2.lbry.com:50001
|
||||
- lbryumx4.lbry.com:50001
|
||||
|
||||
blockchain_name: lbrycrd_main
|
||||
|
||||
data_dir: /home/lbry/.lbrynet
|
||||
download_directory: /home/lbry/downloads
|
||||
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: true
|
||||
|
||||
#components_to_skip:
|
||||
# - hash_announcer
|
||||
# - blob_server
|
||||
# - dht
|
||||
+332
@@ -0,0 +1,332 @@
|
||||
#!/usr/bin/env python
|
||||
"""Bootstrap setuptools installation
|
||||
|
||||
To use setuptools in your package's setup.py, include this
|
||||
file in the same directory and add this to the top of your setup.py::
|
||||
|
||||
from ez_setup import use_setuptools
|
||||
use_setuptools()
|
||||
|
||||
To require a specific version of setuptools, set a download
|
||||
mirror, or use an alternate download directory, simply supply
|
||||
the appropriate options to ``use_setuptools()``.
|
||||
|
||||
This file can also be run as a script to install or upgrade setuptools.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
import optparse
|
||||
import subprocess
|
||||
import platform
|
||||
import textwrap
|
||||
import contextlib
|
||||
|
||||
from distutils import log
|
||||
|
||||
try:
|
||||
from urllib.request import urlopen
|
||||
except ImportError:
|
||||
from urllib2 import urlopen
|
||||
|
||||
try:
|
||||
from site import USER_SITE
|
||||
except ImportError:
|
||||
USER_SITE = None
|
||||
|
||||
DEFAULT_VERSION = "4.0.1"
|
||||
DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/"
|
||||
|
||||
def _python_cmd(*args):
|
||||
"""
|
||||
Return True if the command succeeded.
|
||||
"""
|
||||
args = (sys.executable,) + args
|
||||
return subprocess.call(args) == 0
|
||||
|
||||
|
||||
def _install(archive_filename, install_args=()):
|
||||
with archive_context(archive_filename):
|
||||
# installing
|
||||
log.warn('Installing Setuptools')
|
||||
if not _python_cmd('setup.py', 'install', *install_args):
|
||||
log.warn('Something went wrong during the installation.')
|
||||
log.warn('See the error message above.')
|
||||
# exitcode will be 2
|
||||
return 2
|
||||
|
||||
|
||||
def _build_egg(egg, archive_filename, to_dir):
|
||||
with archive_context(archive_filename):
|
||||
# building an egg
|
||||
log.warn('Building a Setuptools egg in %s', to_dir)
|
||||
_python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
|
||||
# returning the result
|
||||
log.warn(egg)
|
||||
if not os.path.exists(egg):
|
||||
raise IOError('Could not build the egg.')
|
||||
|
||||
|
||||
class ContextualZipFile(zipfile.ZipFile):
|
||||
"""
|
||||
Supplement ZipFile class to support context manager for Python 2.6
|
||||
"""
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, type, value, traceback):
|
||||
self.close()
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
"""
|
||||
Construct a ZipFile or ContextualZipFile as appropriate
|
||||
"""
|
||||
if hasattr(zipfile.ZipFile, '__exit__'):
|
||||
return zipfile.ZipFile(*args, **kwargs)
|
||||
return super(ContextualZipFile, cls).__new__(cls)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def archive_context(filename):
|
||||
# extracting the archive
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
log.warn('Extracting in %s', tmpdir)
|
||||
old_wd = os.getcwd()
|
||||
try:
|
||||
os.chdir(tmpdir)
|
||||
with ContextualZipFile(filename) as archive:
|
||||
archive.extractall()
|
||||
|
||||
# going in the directory
|
||||
subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
|
||||
os.chdir(subdir)
|
||||
log.warn('Now working in %s', subdir)
|
||||
yield
|
||||
|
||||
finally:
|
||||
os.chdir(old_wd)
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
|
||||
def _do_download(version, download_base, to_dir, download_delay):
|
||||
egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg'
|
||||
% (version, sys.version_info[0], sys.version_info[1]))
|
||||
if not os.path.exists(egg):
|
||||
archive = download_setuptools(version, download_base,
|
||||
to_dir, download_delay)
|
||||
_build_egg(egg, archive, to_dir)
|
||||
sys.path.insert(0, egg)
|
||||
|
||||
# Remove previously-imported pkg_resources if present (see
|
||||
# https://bitbucket.org/pypa/setuptools/pull-request/7/ for details).
|
||||
if 'pkg_resources' in sys.modules:
|
||||
del sys.modules['pkg_resources']
|
||||
|
||||
import setuptools
|
||||
setuptools.bootstrap_install_from = egg
|
||||
|
||||
|
||||
def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
|
||||
to_dir=os.curdir, download_delay=15):
|
||||
to_dir = os.path.abspath(to_dir)
|
||||
rep_modules = 'pkg_resources', 'setuptools'
|
||||
imported = set(sys.modules).intersection(rep_modules)
|
||||
try:
|
||||
import pkg_resources
|
||||
except ImportError:
|
||||
return _do_download(version, download_base, to_dir, download_delay)
|
||||
try:
|
||||
pkg_resources.require("setuptools>=" + version)
|
||||
return
|
||||
except pkg_resources.DistributionNotFound:
|
||||
return _do_download(version, download_base, to_dir, download_delay)
|
||||
except pkg_resources.VersionConflict as VC_err:
|
||||
if imported:
|
||||
msg = textwrap.dedent("""
|
||||
The required version of setuptools (>={version}) is not available,
|
||||
and can't be installed while this script is running. Please
|
||||
install a more recent version first, using
|
||||
'easy_install -U setuptools'.
|
||||
|
||||
(Currently using {VC_err.args[0]!r})
|
||||
""").format(VC_err=VC_err, version=version)
|
||||
sys.stderr.write(msg)
|
||||
sys.exit(2)
|
||||
|
||||
# otherwise, reload ok
|
||||
del pkg_resources, sys.modules['pkg_resources']
|
||||
return _do_download(version, download_base, to_dir, download_delay)
|
||||
|
||||
def _clean_check(cmd, target):
|
||||
"""
|
||||
Run the command to download target. If the command fails, clean up before
|
||||
re-raising the error.
|
||||
"""
|
||||
try:
|
||||
subprocess.check_call(cmd)
|
||||
except subprocess.CalledProcessError:
|
||||
if os.access(target, os.F_OK):
|
||||
os.unlink(target)
|
||||
raise
|
||||
|
||||
def download_file_powershell(url, target):
|
||||
"""
|
||||
Download the file at url to target using Powershell (which will validate
|
||||
trust). Raise an exception if the command cannot complete.
|
||||
"""
|
||||
target = os.path.abspath(target)
|
||||
ps_cmd = (
|
||||
"[System.Net.WebRequest]::DefaultWebProxy.Credentials = "
|
||||
"[System.Net.CredentialCache]::DefaultCredentials; "
|
||||
"(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)"
|
||||
% vars()
|
||||
)
|
||||
cmd = [
|
||||
'powershell',
|
||||
'-Command',
|
||||
ps_cmd,
|
||||
]
|
||||
_clean_check(cmd, target)
|
||||
|
||||
def has_powershell():
|
||||
if platform.system() != 'Windows':
|
||||
return False
|
||||
cmd = ['powershell', '-Command', 'echo test']
|
||||
with open(os.path.devnull, 'wb') as devnull:
|
||||
try:
|
||||
subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
download_file_powershell.viable = has_powershell
|
||||
|
||||
def download_file_curl(url, target):
|
||||
cmd = ['curl', url, '--silent', '--output', target]
|
||||
_clean_check(cmd, target)
|
||||
|
||||
def has_curl():
|
||||
cmd = ['curl', '--version']
|
||||
with open(os.path.devnull, 'wb') as devnull:
|
||||
try:
|
||||
subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
download_file_curl.viable = has_curl
|
||||
|
||||
def download_file_wget(url, target):
|
||||
cmd = ['wget', url, '--quiet', '--output-document', target]
|
||||
_clean_check(cmd, target)
|
||||
|
||||
def has_wget():
|
||||
cmd = ['wget', '--version']
|
||||
with open(os.path.devnull, 'wb') as devnull:
|
||||
try:
|
||||
subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
download_file_wget.viable = has_wget
|
||||
|
||||
def download_file_insecure(url, target):
|
||||
"""
|
||||
Use Python to download the file, even though it cannot authenticate the
|
||||
connection.
|
||||
"""
|
||||
src = urlopen(url)
|
||||
try:
|
||||
# Read all the data in one block.
|
||||
data = src.read()
|
||||
finally:
|
||||
src.close()
|
||||
|
||||
# Write all the data in one block to avoid creating a partial file.
|
||||
with open(target, "wb") as dst:
|
||||
dst.write(data)
|
||||
|
||||
download_file_insecure.viable = lambda: True
|
||||
|
||||
def get_best_downloader():
|
||||
downloaders = (
|
||||
download_file_powershell,
|
||||
download_file_curl,
|
||||
download_file_wget,
|
||||
download_file_insecure,
|
||||
)
|
||||
viable_downloaders = (dl for dl in downloaders if dl.viable())
|
||||
return next(viable_downloaders, None)
|
||||
|
||||
def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
|
||||
to_dir=os.curdir, delay=15, downloader_factory=get_best_downloader):
|
||||
"""
|
||||
Download setuptools from a specified location and return its filename
|
||||
|
||||
`version` should be a valid setuptools version number that is available
|
||||
as an egg for download under the `download_base` URL (which should end
|
||||
with a '/'). `to_dir` is the directory where the egg will be downloaded.
|
||||
`delay` is the number of seconds to pause before an actual download
|
||||
attempt.
|
||||
|
||||
``downloader_factory`` should be a function taking no arguments and
|
||||
returning a function for downloading a URL to a target.
|
||||
"""
|
||||
# making sure we use the absolute path
|
||||
to_dir = os.path.abspath(to_dir)
|
||||
zip_name = "setuptools-%s.zip" % version
|
||||
url = download_base + zip_name
|
||||
saveto = os.path.join(to_dir, zip_name)
|
||||
if not os.path.exists(saveto): # Avoid repeated downloads
|
||||
log.warn("Downloading %s", url)
|
||||
downloader = downloader_factory()
|
||||
downloader(url, saveto)
|
||||
return os.path.realpath(saveto)
|
||||
|
||||
def _build_install_args(options):
|
||||
"""
|
||||
Build the arguments to 'python setup.py install' on the setuptools package
|
||||
"""
|
||||
return ['--user'] if options.user_install else []
|
||||
|
||||
def _parse_args():
|
||||
"""
|
||||
Parse the command line for options
|
||||
"""
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option(
|
||||
'--user', dest='user_install', action='store_true', default=False,
|
||||
help='install in user site package (requires Python 2.6 or later)')
|
||||
parser.add_option(
|
||||
'--download-base', dest='download_base', metavar="URL",
|
||||
default=DEFAULT_URL,
|
||||
help='alternative URL from where to download the setuptools package')
|
||||
parser.add_option(
|
||||
'--insecure', dest='downloader_factory', action='store_const',
|
||||
const=lambda: download_file_insecure, default=get_best_downloader,
|
||||
help='Use internal, non-validating downloader'
|
||||
)
|
||||
parser.add_option(
|
||||
'--version', help="Specify which version to download",
|
||||
default=DEFAULT_VERSION,
|
||||
)
|
||||
options, args = parser.parse_args()
|
||||
# positional arguments are ignored
|
||||
return options
|
||||
|
||||
def main():
|
||||
"""Install or upgrade setuptools and EasyInstall"""
|
||||
options = _parse_args()
|
||||
archive = download_setuptools(
|
||||
version=options.version,
|
||||
download_base=options.download_base,
|
||||
downloader_factory=options.downloader_factory,
|
||||
)
|
||||
return _install(archive, _build_install_args(options))
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 97 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.1 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 361 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 5.3 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 31 KiB |
@@ -1,8 +0,0 @@
|
||||
__version__ = "1.0.0"
|
||||
from lbry.wallet import Account, Wallet, WalletManager
|
||||
from lbry.blockchain import Ledger, RegTestLedger, TestNetLedger
|
||||
from lbry.blockchain import Transaction, Output, Input
|
||||
from lbry.blockchain import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc
|
||||
from lbry.service import API, Daemon, FullNode, LightClient
|
||||
from lbry.db.database import Database
|
||||
from lbry.conf import Config
|
||||
@@ -1,6 +0,0 @@
|
||||
from lbry.utils import get_lbry_hash_obj
|
||||
|
||||
MAX_BLOB_SIZE = 2 * 2 ** 20
|
||||
|
||||
# digest_size is in bytes, and blob hashes are hex encoded
|
||||
BLOBHASH_LENGTH = get_lbry_hash_obj().digest_size * 2
|
||||
@@ -1,340 +0,0 @@
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
import contextlib
|
||||
from io import BytesIO
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
|
||||
from lbry.utils import get_lbry_hash_obj
|
||||
from lbry.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
|
||||
|
||||
from lbry.blob import MAX_BLOB_SIZE, BLOBHASH_LENGTH
|
||||
from lbry.blob.blob_info import BlobInfo
|
||||
from lbry.blob.writer import HashBlobWriter
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
HEXMATCH = re.compile("^[a-f,0-9]+$")
|
||||
BACKEND = default_backend()
|
||||
|
||||
|
||||
def is_valid_blobhash(blobhash: str) -> bool:
|
||||
"""Checks whether the blobhash is the correct length and contains only
|
||||
valid characters (0-9, a-f)
|
||||
|
||||
@param blobhash: string, the blobhash to check
|
||||
|
||||
@return: True/False
|
||||
"""
|
||||
return len(blobhash) == BLOBHASH_LENGTH and HEXMATCH.match(blobhash)
|
||||
|
||||
|
||||
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=BACKEND)
|
||||
padder = PKCS7(AES.block_size).padder()
|
||||
encryptor = cipher.encryptor()
|
||||
encrypted = encryptor.update(padder.update(unencrypted) + padder.finalize()) + encryptor.finalize()
|
||||
digest = get_lbry_hash_obj()
|
||||
digest.update(encrypted)
|
||||
return encrypted, digest.hexdigest()
|
||||
|
||||
|
||||
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
|
||||
if len(data) != length:
|
||||
raise ValueError("unexpected length")
|
||||
cipher = Cipher(AES(key), modes.CBC(iv), backend=BACKEND)
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
decryptor = cipher.decryptor()
|
||||
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
|
||||
|
||||
|
||||
class AbstractBlob:
|
||||
"""
|
||||
A chunk of data (up to 2MB) available on the network which is specified by a sha384 hash
|
||||
|
||||
This class is non-io specific
|
||||
"""
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'blob_hash',
|
||||
'length',
|
||||
'blob_completed_callback',
|
||||
'blob_directory',
|
||||
'writers',
|
||||
'verified',
|
||||
'writing',
|
||||
'readers'
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self.loop = loop
|
||||
self.blob_hash = blob_hash
|
||||
self.length = length
|
||||
self.blob_completed_callback = blob_completed_callback
|
||||
self.blob_directory = blob_directory
|
||||
self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
|
||||
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
|
||||
self.readers: typing.List[typing.BinaryIO] = []
|
||||
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise InvalidBlobHashError(blob_hash)
|
||||
|
||||
def __del__(self):
|
||||
if self.writers or self.readers:
|
||||
log.warning("%s not closed before being garbage collected", self.blob_hash)
|
||||
self.close()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
raise NotImplementedError()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError(f"{str(type(self))} not readable, {len(self.readers)} readers {len(self.writers)} writers")
|
||||
with self._reader_context() as reader:
|
||||
try:
|
||||
self.readers.append(reader)
|
||||
yield reader
|
||||
finally:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_length(self, length) -> None:
|
||||
if self.length is not None and length == self.length:
|
||||
return
|
||||
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
|
||||
self.length = length
|
||||
return
|
||||
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
|
||||
|
||||
def get_length(self) -> typing.Optional[int]:
|
||||
return self.length
|
||||
|
||||
def get_is_verified(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def is_readable(self) -> bool:
|
||||
return self.verified.is_set()
|
||||
|
||||
def is_writeable(self) -> bool:
|
||||
return not self.writing.is_set()
|
||||
|
||||
def write_blob(self, blob_bytes: bytes):
|
||||
if not self.is_writeable():
|
||||
raise OSError("cannot open blob for writing")
|
||||
try:
|
||||
self.writing.set()
|
||||
self._write_blob(blob_bytes)
|
||||
finally:
|
||||
self.writing.clear()
|
||||
|
||||
def close(self):
|
||||
while self.writers:
|
||||
_, writer = self.writers.popitem()
|
||||
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
|
||||
writer.finished.cancel()
|
||||
while self.readers:
|
||||
reader = self.readers.pop()
|
||||
if reader:
|
||||
reader.close()
|
||||
|
||||
def delete(self):
|
||||
self.close()
|
||||
self.verified.clear()
|
||||
self.length = None
|
||||
|
||||
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
|
||||
"""
|
||||
Read and send the file to the writer and return the number of bytes sent
|
||||
"""
|
||||
|
||||
if not self.is_readable():
|
||||
raise OSError('blob files cannot be read')
|
||||
with self.reader_context() as handle:
|
||||
try:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
except (ConnectionError, BrokenPipeError, RuntimeError, OSError, AttributeError):
|
||||
return -1
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
"""
|
||||
Decrypt a BlobFile to plaintext bytes
|
||||
"""
|
||||
|
||||
with self.reader_context() as reader:
|
||||
return decrypt_blob_bytes(reader.read(), self.length, key, iv)
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
|
||||
"""
|
||||
Create an encrypted BlobFile from plaintext bytes
|
||||
"""
|
||||
|
||||
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
|
||||
length = len(blob_bytes)
|
||||
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
|
||||
writer = blob.get_blob_writer()
|
||||
writer.write(blob_bytes)
|
||||
await blob.verified.wait()
|
||||
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
|
||||
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
if self.is_writeable():
|
||||
self._write_blob(verified_bytes)
|
||||
self.verified.set()
|
||||
if self.blob_completed_callback:
|
||||
self.blob_completed_callback(self)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
|
||||
raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
|
||||
self.writers[(peer_address, peer_port)] = writer
|
||||
|
||||
def remove_writer(_):
|
||||
if (peer_address, peer_port) in self.writers:
|
||||
del self.writers[(peer_address, peer_port)]
|
||||
|
||||
fut.add_done_callback(remove_writer)
|
||||
|
||||
def writer_finished_callback(finished: asyncio.Future):
|
||||
try:
|
||||
err = finished.exception()
|
||||
if err:
|
||||
raise err
|
||||
verified_bytes = finished.result()
|
||||
while self.writers:
|
||||
_, other = self.writers.popitem()
|
||||
if other is not writer:
|
||||
other.close_handle()
|
||||
self.save_verified_blob(verified_bytes)
|
||||
except (InvalidBlobHashError, InvalidDataError) as error:
|
||||
log.warning("writer error downloading %s: %s", self.blob_hash[:8], str(error))
|
||||
except (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
|
||||
fut.add_done_callback(writer_finished_callback)
|
||||
return writer
|
||||
|
||||
|
||||
class BlobBuffer(AbstractBlob):
|
||||
"""
|
||||
An in-memory only blob
|
||||
"""
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
self._verified_bytes: typing.Optional[BytesIO] = None
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
if not self.is_readable():
|
||||
raise OSError("cannot open blob for reading")
|
||||
try:
|
||||
yield self._verified_bytes
|
||||
finally:
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
self._verified_bytes.close()
|
||||
self._verified_bytes = None
|
||||
return super().delete()
|
||||
|
||||
def __del__(self):
|
||||
super().__del__()
|
||||
if self._verified_bytes:
|
||||
self.delete()
|
||||
|
||||
|
||||
class BlobFile(AbstractBlob):
|
||||
"""
|
||||
A blob existing on the local file system
|
||||
"""
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_hash: str, length: typing.Optional[int] = None,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
|
||||
blob_directory: typing.Optional[str] = None):
|
||||
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
|
||||
if not blob_directory or not os.path.isdir(blob_directory):
|
||||
raise OSError(f"invalid blob directory '{blob_directory}'")
|
||||
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
|
||||
if self.file_exists:
|
||||
file_size = int(os.stat(self.file_path).st_size)
|
||||
if length and length != file_size:
|
||||
log.warning("expected %s to be %s bytes, file has %s", self.blob_hash, length, file_size)
|
||||
self.delete()
|
||||
else:
|
||||
self.length = file_size
|
||||
self.verified.set()
|
||||
|
||||
@property
|
||||
def file_exists(self):
|
||||
return os.path.isfile(self.file_path)
|
||||
|
||||
def is_writeable(self) -> bool:
|
||||
return super().is_writeable() and not os.path.isfile(self.file_path)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[str] = None) -> HashBlobWriter:
|
||||
if self.file_exists:
|
||||
raise OSError(f"File already exists '{self.file_path}'")
|
||||
return super().get_blob_writer(peer_address, peer_port)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
|
||||
handle = open(self.file_path, 'rb')
|
||||
try:
|
||||
yield handle
|
||||
finally:
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
return super().delete()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
cls, loop: asyncio.AbstractEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
|
||||
unencrypted: bytes, blob_num: int,
|
||||
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
|
||||
asyncio.Task]] = None) -> BlobInfo:
|
||||
if not blob_dir or not os.path.isdir(blob_dir):
|
||||
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
|
||||
return await super().create_from_unencrypted(
|
||||
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
|
||||
)
|
||||
@@ -1,26 +0,0 @@
|
||||
import typing
|
||||
|
||||
|
||||
class BlobInfo:
|
||||
__slots__ = [
|
||||
'blob_hash',
|
||||
'blob_num',
|
||||
'length',
|
||||
'iv',
|
||||
]
|
||||
|
||||
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_num = blob_num
|
||||
self.length = length
|
||||
self.iv = iv
|
||||
|
||||
def as_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
'length': self.length,
|
||||
'blob_num': self.blob_num,
|
||||
'iv': self.iv,
|
||||
}
|
||||
if self.blob_hash: # non-terminator blobs have a blob hash
|
||||
d['blob_hash'] = self.blob_hash
|
||||
return d
|
||||
@@ -1,137 +0,0 @@
|
||||
import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.connection_manager import ConnectionManager
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.dht.protocol.data_store import DictDataStore
|
||||
|
||||
|
||||
class SQLiteStorage:
|
||||
pass
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
|
||||
node_data_store: typing.Optional['DictDataStore'] = None):
|
||||
"""
|
||||
This class stores blobs on the hard disk
|
||||
|
||||
blob_dir - directory where blobs are stored
|
||||
storage - SQLiteStorage object
|
||||
"""
|
||||
self.loop = loop
|
||||
self.blob_dir = blob_dir
|
||||
self.storage = storage
|
||||
self._node_data_store = node_data_store
|
||||
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
|
||||
if self.config.save_blobs or (
|
||||
is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash))):
|
||||
return BlobFile(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
return BlobBuffer(
|
||||
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
|
||||
)
|
||||
|
||||
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
|
||||
if blob_hash in self.blobs:
|
||||
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
|
||||
buffer = self.blobs.pop(blob_hash)
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
if buffer.is_readable():
|
||||
with buffer.reader_context() as reader:
|
||||
self.blobs[blob_hash].write_blob(reader.read())
|
||||
if length and self.blobs[blob_hash].length is None:
|
||||
self.blobs[blob_hash].set_length(length)
|
||||
else:
|
||||
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
|
||||
return self.blobs[blob_hash]
|
||||
|
||||
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise ValueError(blob_hash)
|
||||
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
return False
|
||||
if blob_hash in self.blobs:
|
||||
return self.blobs[blob_hash].get_is_verified()
|
||||
return self._get_blob(blob_hash, length).get_is_verified()
|
||||
|
||||
async def setup(self) -> bool:
|
||||
def get_files_in_blob_dir() -> typing.Set[str]:
|
||||
if not self.blob_dir:
|
||||
return set()
|
||||
return {
|
||||
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
|
||||
}
|
||||
|
||||
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
|
||||
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
|
||||
if to_add:
|
||||
self.completed_blob_hashes.update(to_add)
|
||||
if self.config.track_bandwidth:
|
||||
self.connection_manager.start()
|
||||
return True
|
||||
|
||||
def stop(self):
|
||||
self.connection_manager.stop()
|
||||
while self.blobs:
|
||||
_, blob = self.blobs.popitem()
|
||||
blob.close()
|
||||
self.completed_blob_hashes.clear()
|
||||
|
||||
def get_stream_descriptor(self, sd_hash):
|
||||
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
|
||||
|
||||
def blob_completed(self, blob: AbstractBlob) -> asyncio.Task:
|
||||
if blob.blob_hash is None:
|
||||
raise Exception("Blob hash is None")
|
||||
if not blob.length:
|
||||
raise Exception("Blob has a length of 0")
|
||||
if isinstance(blob, BlobFile):
|
||||
if blob.blob_hash not in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.add(blob.blob_hash)
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
|
||||
else:
|
||||
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
|
||||
|
||||
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
|
||||
|
||||
def delete_blob(self, blob_hash: str):
|
||||
if not is_valid_blobhash(blob_hash):
|
||||
raise Exception("invalid blob hash to delete")
|
||||
|
||||
if blob_hash not in self.blobs:
|
||||
if self.blob_dir and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
|
||||
os.remove(os.path.join(self.blob_dir, blob_hash))
|
||||
else:
|
||||
self.blobs.pop(blob_hash).delete()
|
||||
if blob_hash in self.completed_blob_hashes:
|
||||
self.completed_blob_hashes.remove(blob_hash)
|
||||
|
||||
async def delete_blobs(self, blob_hashes: typing.List[str], delete_from_db: typing.Optional[bool] = True):
|
||||
for blob_hash in blob_hashes:
|
||||
self.delete_blob(blob_hash)
|
||||
|
||||
if delete_from_db:
|
||||
await self.storage.delete_blobs_from_db(blob_hashes)
|
||||
@@ -1,68 +0,0 @@
|
||||
import typing
|
||||
import logging
|
||||
import asyncio
|
||||
from io import BytesIO
|
||||
from lbry.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbry.utils import get_lbry_hash_obj
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HashBlobWriter:
|
||||
def __init__(self, expected_blob_hash: str, get_length: typing.Callable[[], int],
|
||||
finished: asyncio.Future):
|
||||
self.expected_blob_hash = expected_blob_hash
|
||||
self.get_length = get_length
|
||||
self.buffer = BytesIO()
|
||||
self.finished = finished
|
||||
self.finished.add_done_callback(lambda *_: self.close_handle())
|
||||
self._hashsum = get_lbry_hash_obj()
|
||||
self.len_so_far = 0
|
||||
|
||||
def __del__(self):
|
||||
if self.buffer is not None:
|
||||
log.warning("Garbage collection was called, but writer was not closed yet")
|
||||
self.close_handle()
|
||||
|
||||
def calculate_blob_hash(self) -> str:
|
||||
return self._hashsum.hexdigest()
|
||||
|
||||
def closed(self):
|
||||
return self.buffer is None or self.buffer.closed
|
||||
|
||||
def write(self, data: bytes):
|
||||
expected_length = self.get_length()
|
||||
if not expected_length:
|
||||
raise OSError("unknown blob length")
|
||||
if self.buffer is None:
|
||||
log.warning("writer has already been closed")
|
||||
if not self.finished.done():
|
||||
self.finished.cancel()
|
||||
return
|
||||
raise OSError('I/O operation on closed file')
|
||||
|
||||
self._hashsum.update(data)
|
||||
self.len_so_far += len(data)
|
||||
if self.len_so_far > expected_length:
|
||||
self.finished.set_exception(InvalidDataError(
|
||||
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
|
||||
))
|
||||
self.close_handle()
|
||||
return
|
||||
self.buffer.write(data)
|
||||
if self.len_so_far == expected_length:
|
||||
blob_hash = self.calculate_blob_hash()
|
||||
if blob_hash != self.expected_blob_hash:
|
||||
self.finished.set_exception(InvalidBlobHashError(
|
||||
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
|
||||
))
|
||||
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
|
||||
self.finished.set_result(self.buffer.getvalue())
|
||||
self.close_handle()
|
||||
|
||||
def close_handle(self):
|
||||
if not self.finished.done():
|
||||
self.finished.cancel()
|
||||
if self.buffer is not None:
|
||||
self.buffer.close()
|
||||
self.buffer = None
|
||||
@@ -1,253 +0,0 @@
|
||||
import asyncio
|
||||
import time
|
||||
import logging
|
||||
import typing
|
||||
import binascii
|
||||
from typing import Optional
|
||||
from lbry.error import InvalidBlobHashError, InvalidDataError
|
||||
from lbry.blob_exchange.serialization import BlobResponse, BlobRequest
|
||||
from lbry.utils import cache_concurrent
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.blob.blob_file import AbstractBlob
|
||||
from lbry.blob.writer import HashBlobWriter
|
||||
from lbry.connection_manager import ConnectionManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_timeout: typing.Optional[float] = 10,
|
||||
connection_manager: typing.Optional['ConnectionManager'] = None):
|
||||
self.loop = loop
|
||||
self.peer_port: typing.Optional[int] = None
|
||||
self.peer_address: typing.Optional[str] = None
|
||||
self.transport: typing.Optional[asyncio.Transport] = None
|
||||
self.peer_timeout = peer_timeout
|
||||
self.connection_manager = connection_manager
|
||||
self.writer: typing.Optional['HashBlobWriter'] = None
|
||||
self.blob: typing.Optional['AbstractBlob'] = None
|
||||
|
||||
self._blob_bytes_received = 0
|
||||
self._response_fut: typing.Optional[asyncio.Future] = None
|
||||
self.buf = b''
|
||||
|
||||
# this is here to handle the race when the downloader is closed right as response_fut gets a result
|
||||
self.closed = asyncio.Event(loop=self.loop)
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
if self.connection_manager:
|
||||
if not self.peer_address:
|
||||
addr_info = self.transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr_info
|
||||
# assert self.peer_address is not None
|
||||
self.connection_manager.received_data(f"{self.peer_address}:{self.peer_port}", len(data))
|
||||
if not self.transport or self.transport.is_closing():
|
||||
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
|
||||
binascii.hexlify(data))
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
return
|
||||
if not self._response_fut:
|
||||
log.warning("Protocol received data before expected, probable race on keep alive. Closing transport.")
|
||||
return self.close()
|
||||
if self._blob_bytes_received and not self.writer.closed():
|
||||
return self._write(data)
|
||||
|
||||
response = BlobResponse.deserialize(self.buf + data)
|
||||
if not response.responses and not self._response_fut.done():
|
||||
self.buf += data
|
||||
return
|
||||
else:
|
||||
self.buf = b''
|
||||
|
||||
if response.responses and self.blob:
|
||||
blob_response = response.get_blob_response()
|
||||
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
|
||||
# set the expected length for the incoming blob if we didn't know it
|
||||
self.blob.set_length(blob_response.length)
|
||||
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
|
||||
# the server started sending a blob we didn't request
|
||||
log.warning("%s started sending blob we didn't request %s instead of %s", self.peer_address,
|
||||
blob_response.blob_hash, self.blob.blob_hash)
|
||||
return
|
||||
if response.responses:
|
||||
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
|
||||
# fire the Future with the response to our request
|
||||
self._response_fut.set_result(response)
|
||||
if response.blob_data and self.writer and not self.writer.closed():
|
||||
# log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
|
||||
# write blob bytes if we're writing a blob and have blob bytes to write
|
||||
self._write(response.blob_data)
|
||||
|
||||
def _write(self, data: bytes):
|
||||
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
|
||||
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
|
||||
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
|
||||
self._blob_bytes_received += len(data)
|
||||
try:
|
||||
self.writer.write(data)
|
||||
except OSError as err:
|
||||
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
except asyncio.TimeoutError as err:
|
||||
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.set_exception(err)
|
||||
|
||||
async def _download_blob(self) -> typing.Tuple[int, Optional['BlobExchangeClientProtocol']]: # pylint: disable=too-many-return-statements
|
||||
"""
|
||||
:return: download success (bool), connected protocol (BlobExchangeClientProtocol)
|
||||
"""
|
||||
start_time = time.perf_counter()
|
||||
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
|
||||
blob_hash = self.blob.blob_hash
|
||||
if not self.peer_address:
|
||||
addr_info = self.transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr_info
|
||||
try:
|
||||
msg = request.serialize()
|
||||
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
|
||||
self.transport.write(msg)
|
||||
if self.connection_manager:
|
||||
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
|
||||
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
|
||||
availability_response = response.get_availability_response()
|
||||
price_response = response.get_price_response()
|
||||
blob_response = response.get_blob_response()
|
||||
if self.closed.is_set():
|
||||
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
|
||||
log.warning(msg)
|
||||
raise asyncio.CancelledError(msg)
|
||||
if (not blob_response or blob_response.error) and\
|
||||
(not availability_response or not availability_response.available_blobs):
|
||||
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
|
||||
self.peer_port)
|
||||
log.warning(response.to_dict())
|
||||
return self._blob_bytes_received, self.close()
|
||||
elif availability_response and availability_response.available_blobs and \
|
||||
availability_response.available_blobs != [self.blob.blob_hash]:
|
||||
log.warning("blob availability response doesn't match our request from %s:%i",
|
||||
self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
elif not availability_response:
|
||||
log.warning("response from %s:%i did not include an availability response (we requested %s)",
|
||||
self.peer_address, self.peer_port, blob_hash)
|
||||
return self._blob_bytes_received, self.close()
|
||||
|
||||
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
|
||||
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not blob_response or blob_response.error:
|
||||
log.warning("blob can't be downloaded from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
|
||||
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
if self.blob.length is not None and self.blob.length != blob_response.length:
|
||||
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
|
||||
f" timeout in {self.peer_timeout}"
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
log.info("%s at %fMB/s", msg,
|
||||
round((float(self._blob_bytes_received) /
|
||||
float(time.perf_counter() - start_time)) / 1000000.0, 2))
|
||||
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
|
||||
return self._blob_bytes_received, self
|
||||
except asyncio.TimeoutError:
|
||||
return self._blob_bytes_received, self.close()
|
||||
except (InvalidBlobHashError, InvalidDataError):
|
||||
log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port)
|
||||
return self._blob_bytes_received, self.close()
|
||||
|
||||
def close(self):
|
||||
self.closed.set()
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
self._response_fut = None
|
||||
self.writer = None
|
||||
self.blob = None
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
self.transport = None
|
||||
self.buf = b''
|
||||
|
||||
async def download_blob(self, blob: 'AbstractBlob') -> typing.Tuple[int, Optional['BlobExchangeClientProtocol']]:
|
||||
self.closed.clear()
|
||||
blob_hash = blob.blob_hash
|
||||
if blob.get_is_verified() or not blob.is_writeable():
|
||||
return 0, self
|
||||
try:
|
||||
self._blob_bytes_received = 0
|
||||
self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
|
||||
self._response_fut = asyncio.Future(loop=self.loop)
|
||||
return await self._download_blob()
|
||||
except OSError:
|
||||
# i'm not sure how to fix this race condition - jack
|
||||
log.warning("race happened downloading %s from %s:%s", blob_hash, self.peer_address, self.peer_port)
|
||||
# return self._blob_bytes_received, self.transport
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
if self._response_fut and not self._response_fut.done():
|
||||
self._response_fut.cancel()
|
||||
self.close()
|
||||
return self._blob_bytes_received, None
|
||||
except asyncio.CancelledError:
|
||||
self.close()
|
||||
raise
|
||||
finally:
|
||||
if self.writer and not self.writer.closed():
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
|
||||
def connection_made(self, transport: asyncio.Transport):
|
||||
addr = transport.get_extra_info('peername')
|
||||
self.peer_address, self.peer_port = addr[0], addr[1]
|
||||
self.transport = transport
|
||||
if self.connection_manager:
|
||||
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
|
||||
|
||||
def connection_lost(self, exc):
|
||||
if self.connection_manager:
|
||||
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
|
||||
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(exc),
|
||||
str(type(exc)))
|
||||
self.close()
|
||||
|
||||
|
||||
@cache_concurrent
|
||||
async def request_blob(loop: asyncio.AbstractEventLoop, blob: Optional['AbstractBlob'], address: str,
|
||||
tcp_port: int, peer_connect_timeout: float, blob_download_timeout: float,
|
||||
connected_protocol: Optional['BlobExchangeClientProtocol'] = None,
|
||||
connection_id: int = 0, connection_manager: Optional['ConnectionManager'] = None)\
|
||||
-> typing.Tuple[int, Optional['BlobExchangeClientProtocol']]:
|
||||
"""
|
||||
Returns [<amount of bytes received>, <client protocol if connected>]
|
||||
"""
|
||||
|
||||
protocol = connected_protocol
|
||||
if not connected_protocol or not connected_protocol.transport or connected_protocol.transport.is_closing():
|
||||
connected_protocol = None
|
||||
protocol = BlobExchangeClientProtocol(
|
||||
loop, blob_download_timeout, connection_manager
|
||||
)
|
||||
else:
|
||||
log.debug("reusing connection for %s:%d", address, tcp_port)
|
||||
try:
|
||||
if not connected_protocol:
|
||||
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
|
||||
peer_connect_timeout, loop=loop)
|
||||
connected_protocol = protocol
|
||||
if blob is None or blob.get_is_verified() or not blob.is_writeable():
|
||||
# blob is None happens when we are just opening a connection
|
||||
# file exists but not verified means someone is writing right now, give it time, come back later
|
||||
return 0, connected_protocol
|
||||
return await connected_protocol.download_blob(blob)
|
||||
except (asyncio.TimeoutError, ConnectionRefusedError, ConnectionAbortedError, OSError):
|
||||
return 0, None
|
||||
@@ -1,147 +0,0 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
from lbry.utils import cache_concurrent
|
||||
from lbry.blob_exchange.client import request_blob
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.dht.node import Node
|
||||
from lbry.dht.peer import KademliaPeer
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob.blob_file import AbstractBlob
|
||||
from lbry.blob_exchange.client import BlobExchangeClientProtocol
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobDownloader:
|
||||
BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
peer_queue: asyncio.Queue):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_queue = peer_queue
|
||||
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
|
||||
self.ignored: typing.Dict['KademliaPeer', int] = {}
|
||||
self.scores: typing.Dict['KademliaPeer', int] = {}
|
||||
self.failures: typing.Dict['KademliaPeer', int] = {}
|
||||
self.connection_failures: typing.Set['KademliaPeer'] = set()
|
||||
self.connections: typing.Dict['KademliaPeer', 'BlobExchangeClientProtocol'] = {}
|
||||
self.is_running = asyncio.Event(loop=self.loop)
|
||||
|
||||
def should_race_continue(self, blob: 'AbstractBlob'):
|
||||
max_probes = self.config.max_connections_per_download * (1 if self.connections else 10)
|
||||
if len(self.active_connections) >= max_probes:
|
||||
return False
|
||||
return not (blob.get_is_verified() or not blob.is_writeable())
|
||||
|
||||
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0,
|
||||
just_probe: bool = False):
|
||||
if blob.get_is_verified():
|
||||
return
|
||||
start = self.loop.time()
|
||||
bytes_received, protocol = await request_blob(
|
||||
self.loop, blob if not just_probe else None, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
|
||||
self.config.blob_download_timeout, connected_protocol=self.connections.get(peer),
|
||||
connection_id=connection_id, connection_manager=self.blob_manager.connection_manager
|
||||
)
|
||||
if not bytes_received and not protocol and peer not in self.connection_failures:
|
||||
self.connection_failures.add(peer)
|
||||
if not protocol and peer not in self.ignored:
|
||||
self.ignored[peer] = self.loop.time()
|
||||
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = self.failures.get(peer, 0) + 1
|
||||
if peer in self.connections:
|
||||
del self.connections[peer]
|
||||
elif protocol:
|
||||
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
|
||||
self.failures[peer] = 0
|
||||
self.connections[peer] = protocol
|
||||
elapsed = self.loop.time() - start
|
||||
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1
|
||||
|
||||
async def new_peer_or_finished(self):
|
||||
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
|
||||
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
|
||||
|
||||
def cleanup_active(self):
|
||||
if not self.active_connections and not self.connections:
|
||||
self.clearbanned()
|
||||
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
|
||||
for peer in to_remove:
|
||||
del self.active_connections[peer]
|
||||
|
||||
def clearbanned(self):
|
||||
now = self.loop.time()
|
||||
self.ignored = {
|
||||
peer: when for (peer, when) in self.ignored.items()
|
||||
if (now - when) < min(30.0, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
|
||||
}
|
||||
|
||||
@cache_concurrent
|
||||
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None,
|
||||
connection_id: int = 0) -> 'AbstractBlob':
|
||||
blob = self.blob_manager.get_blob(blob_hash, length)
|
||||
if blob.get_is_verified():
|
||||
return blob
|
||||
self.is_running.set()
|
||||
tried_for_this_blob: typing.Set['KademliaPeer'] = set()
|
||||
try:
|
||||
while not blob.get_is_verified() and self.is_running.is_set():
|
||||
batch: typing.Set['KademliaPeer'] = set(self.connections.keys())
|
||||
while not self.peer_queue.empty():
|
||||
batch.update(self.peer_queue.get_nowait())
|
||||
log.debug(
|
||||
"%s running, %d peers, %d ignored, %d active, %s connections", blob_hash[:6],
|
||||
len(batch), len(self.ignored), len(self.active_connections), len(self.connections)
|
||||
)
|
||||
re_add: typing.Set['KademliaPeer'] = set()
|
||||
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
|
||||
if peer in self.ignored:
|
||||
continue
|
||||
if peer in tried_for_this_blob:
|
||||
continue
|
||||
if peer in self.active_connections:
|
||||
if peer not in re_add:
|
||||
re_add.add(peer)
|
||||
continue
|
||||
if not self.should_race_continue(blob):
|
||||
break
|
||||
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
|
||||
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
|
||||
self.active_connections[peer] = t
|
||||
tried_for_this_blob.add(peer)
|
||||
if not re_add:
|
||||
self.peer_queue.put_nowait(list(batch))
|
||||
await self.new_peer_or_finished()
|
||||
self.cleanup_active()
|
||||
log.debug("downloaded %s", blob_hash[:8])
|
||||
return blob
|
||||
finally:
|
||||
blob.close()
|
||||
if self.loop.is_running():
|
||||
self.loop.call_soon(self.cleanup_active)
|
||||
|
||||
def close(self):
|
||||
self.connection_failures.clear()
|
||||
self.scores.clear()
|
||||
self.ignored.clear()
|
||||
self.is_running.clear()
|
||||
for protocol in self.connections.values():
|
||||
protocol.close()
|
||||
|
||||
|
||||
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
|
||||
blob_hash: str) -> 'AbstractBlob':
|
||||
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
|
||||
search_queue.put_nowait(blob_hash)
|
||||
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
|
||||
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
|
||||
try:
|
||||
return await downloader.download_blob(blob_hash)
|
||||
finally:
|
||||
if accumulate_task and not accumulate_task.done():
|
||||
accumulate_task.cancel()
|
||||
downloader.close()
|
||||
@@ -1,282 +0,0 @@
|
||||
import typing
|
||||
import json
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BlobMessage:
|
||||
key = ''
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class BlobPriceRequest(BlobMessage):
|
||||
key = 'blob_data_payment_rate'
|
||||
|
||||
def __init__(self, blob_data_payment_rate: float, **kwargs) -> None:
|
||||
self.blob_data_payment_rate = blob_data_payment_rate
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.blob_data_payment_rate
|
||||
}
|
||||
|
||||
|
||||
class BlobPriceResponse(BlobMessage):
|
||||
key = 'blob_data_payment_rate'
|
||||
rate_accepted = 'RATE_ACCEPTED'
|
||||
rate_too_low = 'RATE_TOO_LOW'
|
||||
rate_unset = 'RATE_UNSET'
|
||||
|
||||
def __init__(self, blob_data_payment_rate: str, **kwargs) -> None:
|
||||
if blob_data_payment_rate not in (self.rate_accepted, self.rate_too_low, self.rate_unset):
|
||||
raise ValueError(blob_data_payment_rate)
|
||||
self.blob_data_payment_rate = blob_data_payment_rate
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.blob_data_payment_rate
|
||||
}
|
||||
|
||||
|
||||
class BlobAvailabilityRequest(BlobMessage):
|
||||
key = 'requested_blobs'
|
||||
|
||||
def __init__(self, requested_blobs: typing.List[str], lbrycrd_address: typing.Optional[bool] = True,
|
||||
**kwargs) -> None:
|
||||
assert len(requested_blobs) > 0
|
||||
self.requested_blobs = requested_blobs
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.requested_blobs,
|
||||
'lbrycrd_address': self.lbrycrd_address
|
||||
}
|
||||
|
||||
|
||||
class BlobAvailabilityResponse(BlobMessage):
|
||||
key = 'available_blobs'
|
||||
|
||||
def __init__(self, available_blobs: typing.List[str], lbrycrd_address: typing.Optional[str] = True,
|
||||
**kwargs) -> None:
|
||||
self.available_blobs = available_blobs
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
d = {
|
||||
self.key: self.available_blobs
|
||||
}
|
||||
if self.lbrycrd_address:
|
||||
d['lbrycrd_address'] = self.lbrycrd_address
|
||||
return d
|
||||
|
||||
|
||||
class BlobDownloadRequest(BlobMessage):
|
||||
key = 'requested_blob'
|
||||
|
||||
def __init__(self, requested_blob: str, **kwargs) -> None:
|
||||
self.requested_blob = requested_blob
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.requested_blob
|
||||
}
|
||||
|
||||
|
||||
class BlobDownloadResponse(BlobMessage):
|
||||
key = 'incoming_blob'
|
||||
|
||||
def __init__(self, **response: typing.Dict) -> None:
|
||||
incoming_blob = response[self.key]
|
||||
self.error = None
|
||||
self.incoming_blob = None
|
||||
if 'error' in incoming_blob:
|
||||
self.error = incoming_blob['error']
|
||||
else:
|
||||
self.incoming_blob = {'blob_hash': incoming_blob['blob_hash'], 'length': incoming_blob['length']}
|
||||
self.length = None if not self.incoming_blob else self.incoming_blob['length']
|
||||
self.blob_hash = None if not self.incoming_blob else self.incoming_blob['blob_hash']
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key if not self.error else 'error': self.incoming_blob or self.error,
|
||||
}
|
||||
|
||||
|
||||
class BlobPaymentAddressRequest(BlobMessage):
|
||||
key = 'lbrycrd_address'
|
||||
|
||||
def __init__(self, lbrycrd_address: str, **kwargs) -> None:
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.lbrycrd_address
|
||||
}
|
||||
|
||||
|
||||
class BlobPaymentAddressResponse(BlobPaymentAddressRequest):
|
||||
pass
|
||||
|
||||
|
||||
class BlobErrorResponse(BlobMessage):
|
||||
key = 'error'
|
||||
|
||||
def __init__(self, error: str, **kwargs) -> None:
|
||||
self.error = error
|
||||
|
||||
def to_dict(self) -> typing.Dict:
|
||||
return {
|
||||
self.key: self.error
|
||||
}
|
||||
|
||||
|
||||
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest, # pylint: disable=invalid-name
|
||||
BlobPaymentAddressRequest]
|
||||
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse, # pylint: disable=invalid-name
|
||||
BlobErrorResponse, BlobPaymentAddressResponse]
|
||||
|
||||
|
||||
def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[typing.Dict], bytes]:
|
||||
# scenarios:
|
||||
# <json>
|
||||
# <blob bytes>
|
||||
# <json><blob bytes>
|
||||
|
||||
curr_pos = 0
|
||||
while True:
|
||||
next_close_paren = response_msg.find(b'}', curr_pos)
|
||||
if next_close_paren == -1:
|
||||
return None, response_msg
|
||||
curr_pos = next_close_paren + 1
|
||||
try:
|
||||
response = json.loads(response_msg[:curr_pos])
|
||||
except ValueError:
|
||||
continue
|
||||
possible_response_keys = {
|
||||
BlobPaymentAddressResponse.key,
|
||||
BlobAvailabilityResponse.key,
|
||||
BlobPriceResponse.key,
|
||||
BlobDownloadResponse.key
|
||||
}
|
||||
if isinstance(response, dict) and response.keys():
|
||||
if set(response.keys()).issubset(possible_response_keys):
|
||||
return response, response_msg[curr_pos:]
|
||||
return None, response_msg
|
||||
|
||||
|
||||
class BlobRequest:
|
||||
def __init__(self, requests: typing.List[blob_request_types]) -> None:
|
||||
self.requests = requests
|
||||
|
||||
def to_dict(self):
|
||||
d = {}
|
||||
for request in self.requests:
|
||||
d.update(request.to_dict())
|
||||
return d
|
||||
|
||||
def _get_request(self, request_type: blob_request_types):
|
||||
request = tuple(filter(lambda r: type(r) == request_type, self.requests)) # pylint: disable=unidiomatic-typecheck
|
||||
if request:
|
||||
return request[0]
|
||||
|
||||
def get_availability_request(self) -> typing.Optional[BlobAvailabilityRequest]:
|
||||
response = self._get_request(BlobAvailabilityRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_price_request(self) -> typing.Optional[BlobPriceRequest]:
|
||||
response = self._get_request(BlobPriceRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_blob_request(self) -> typing.Optional[BlobDownloadRequest]:
|
||||
response = self._get_request(BlobDownloadRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_address_request(self) -> typing.Optional[BlobPaymentAddressRequest]:
|
||||
response = self._get_request(BlobPaymentAddressRequest)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def serialize(self) -> bytes:
|
||||
return json.dumps(self.to_dict()).encode()
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, data: bytes) -> 'BlobRequest':
|
||||
request = json.loads(data)
|
||||
return cls([
|
||||
request_type(**request)
|
||||
for request_type in (BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
|
||||
BlobPaymentAddressRequest)
|
||||
if request_type.key in request
|
||||
])
|
||||
|
||||
@classmethod
|
||||
def make_request_for_blob_hash(cls, blob_hash: str) -> 'BlobRequest':
|
||||
return cls(
|
||||
[BlobAvailabilityRequest([blob_hash]), BlobPriceRequest(0.0), BlobDownloadRequest(blob_hash)]
|
||||
)
|
||||
|
||||
|
||||
class BlobResponse:
|
||||
def __init__(self, responses: typing.List[blob_response_types], blob_data: typing.Optional[bytes] = None) -> None:
|
||||
self.responses = responses
|
||||
self.blob_data = blob_data
|
||||
|
||||
def to_dict(self):
|
||||
d = {}
|
||||
for response in self.responses:
|
||||
d.update(response.to_dict())
|
||||
return d
|
||||
|
||||
def _get_response(self, response_type: blob_response_types):
|
||||
response = tuple(filter(lambda r: type(r) == response_type, self.responses)) # pylint: disable=unidiomatic-typecheck
|
||||
if response:
|
||||
return response[0]
|
||||
|
||||
def get_error_response(self) -> typing.Optional[BlobErrorResponse]:
|
||||
error = self._get_response(BlobErrorResponse)
|
||||
if error:
|
||||
log.error(error)
|
||||
return error
|
||||
|
||||
def get_availability_response(self) -> typing.Optional[BlobAvailabilityResponse]:
|
||||
response = self._get_response(BlobAvailabilityResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_price_response(self) -> typing.Optional[BlobPriceResponse]:
|
||||
response = self._get_response(BlobPriceResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_blob_response(self) -> typing.Optional[BlobDownloadResponse]:
|
||||
response = self._get_response(BlobDownloadResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def get_address_response(self) -> typing.Optional[BlobPaymentAddressResponse]:
|
||||
response = self._get_response(BlobPaymentAddressResponse)
|
||||
if response:
|
||||
return response
|
||||
|
||||
def serialize(self) -> bytes:
|
||||
return json.dumps(self.to_dict()).encode()
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, data: bytes) -> 'BlobResponse':
|
||||
response, extra = _parse_blob_response(data)
|
||||
requests = []
|
||||
if response:
|
||||
requests.extend([
|
||||
response_type(**response)
|
||||
for response_type in (BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
|
||||
BlobErrorResponse, BlobPaymentAddressResponse)
|
||||
if response_type.key in response
|
||||
])
|
||||
return cls(requests, extra)
|
||||
@@ -1,186 +0,0 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import typing
|
||||
from json.decoder import JSONDecodeError
|
||||
from lbry.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
|
||||
from lbry.blob_exchange.serialization import BlobAvailabilityResponse, BlobPriceResponse, BlobDownloadResponse, \
|
||||
BlobPaymentAddressResponse
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# a standard request will be 295 bytes
|
||||
MAX_REQUEST_SIZE = 1200
|
||||
|
||||
|
||||
class BlobServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str,
|
||||
idle_timeout: float = 30.0, transfer_timeout: float = 60.0):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.idle_timeout = idle_timeout
|
||||
self.transfer_timeout = transfer_timeout
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.buf = b''
|
||||
self.transport: typing.Optional[asyncio.Transport] = None
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.peer_address_and_port: typing.Optional[str] = None
|
||||
self.started_transfer = asyncio.Event(loop=self.loop)
|
||||
self.transfer_finished = asyncio.Event(loop=self.loop)
|
||||
self.close_on_idle_task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
async def close_on_idle(self):
|
||||
while self.transport:
|
||||
try:
|
||||
await asyncio.wait_for(self.started_transfer.wait(), self.idle_timeout, loop=self.loop)
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("closing idle connection from %s", self.peer_address_and_port)
|
||||
return self.close()
|
||||
self.started_transfer.clear()
|
||||
await self.transfer_finished.wait()
|
||||
self.transfer_finished.clear()
|
||||
|
||||
def close(self):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
self.close_on_idle_task = self.loop.create_task(self.close_on_idle())
|
||||
self.peer_address_and_port = "%s:%i" % self.transport.get_extra_info('peername')
|
||||
self.blob_manager.connection_manager.connection_received(self.peer_address_and_port)
|
||||
log.debug("received connection from %s", self.peer_address_and_port)
|
||||
|
||||
def connection_lost(self, exc: typing.Optional[Exception]) -> None:
|
||||
log.debug("lost connection from %s", self.peer_address_and_port)
|
||||
self.blob_manager.connection_manager.incoming_connection_lost(self.peer_address_and_port)
|
||||
self.transport = None
|
||||
if self.close_on_idle_task and not self.close_on_idle_task.done():
|
||||
self.close_on_idle_task.cancel()
|
||||
self.close_on_idle_task = None
|
||||
|
||||
def send_response(self, responses: typing.List[blob_response_types]):
|
||||
to_send = []
|
||||
while responses:
|
||||
to_send.append(responses.pop())
|
||||
serialized = BlobResponse(to_send).serialize()
|
||||
self.transport.write(serialized)
|
||||
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, len(serialized))
|
||||
|
||||
async def handle_request(self, request: BlobRequest):
|
||||
addr = self.transport.get_extra_info('peername')
|
||||
peer_address, peer_port = addr
|
||||
|
||||
responses = []
|
||||
address_request = request.get_address_request()
|
||||
if address_request:
|
||||
responses.append(BlobPaymentAddressResponse(lbrycrd_address=self.lbrycrd_address))
|
||||
availability_request = request.get_availability_request()
|
||||
if availability_request:
|
||||
responses.append(BlobAvailabilityResponse(available_blobs=list(set(
|
||||
filter(lambda blob_hash: blob_hash in self.blob_manager.completed_blob_hashes,
|
||||
availability_request.requested_blobs)
|
||||
))))
|
||||
price_request = request.get_price_request()
|
||||
if price_request:
|
||||
responses.append(BlobPriceResponse(blob_data_payment_rate='RATE_ACCEPTED'))
|
||||
download_request = request.get_blob_request()
|
||||
|
||||
if download_request:
|
||||
blob = self.blob_manager.get_blob(download_request.requested_blob)
|
||||
if blob.get_is_verified():
|
||||
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
|
||||
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
|
||||
self.send_response(responses)
|
||||
blob_hash = blob.blob_hash[:8]
|
||||
log.debug("send %s to %s:%i", blob_hash, peer_address, peer_port)
|
||||
self.started_transfer.set()
|
||||
try:
|
||||
sent = await asyncio.wait_for(blob.sendfile(self), self.transfer_timeout, loop=self.loop)
|
||||
if sent and sent > 0:
|
||||
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent)
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob_hash, sent, peer_address, peer_port)
|
||||
else:
|
||||
self.close()
|
||||
log.debug("stopped sending %s to %s:%i", blob_hash, peer_address, peer_port)
|
||||
return
|
||||
except (OSError, ValueError, asyncio.TimeoutError) as err:
|
||||
if isinstance(err, asyncio.TimeoutError):
|
||||
log.debug("timed out sending blob %s to %s", blob_hash, peer_address)
|
||||
else:
|
||||
log.warning("could not read blob %s to send %s:%i", blob_hash, peer_address, peer_port)
|
||||
self.close()
|
||||
return
|
||||
finally:
|
||||
self.transfer_finished.set()
|
||||
else:
|
||||
log.info("don't have %s to send %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
if responses and not self.transport.is_closing():
|
||||
self.send_response(responses)
|
||||
|
||||
def data_received(self, data):
|
||||
request = None
|
||||
if len(self.buf) + len(data or b'') >= MAX_REQUEST_SIZE:
|
||||
log.warning("request from %s is too large", self.peer_address_and_port)
|
||||
self.close()
|
||||
return
|
||||
if data:
|
||||
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
|
||||
_, separator, remainder = data.rpartition(b'}')
|
||||
if not separator:
|
||||
self.buf += data
|
||||
return
|
||||
try:
|
||||
request = BlobRequest.deserialize(self.buf + data)
|
||||
self.buf = remainder
|
||||
except JSONDecodeError:
|
||||
log.error("request from %s is not valid json (%i bytes): %s", self.peer_address_and_port,
|
||||
len(self.buf + data), '' if not data else binascii.hexlify(self.buf + data).decode())
|
||||
self.close()
|
||||
return
|
||||
if not request.requests:
|
||||
log.error("failed to decode request from %s (%i bytes): %s", self.peer_address_and_port,
|
||||
len(self.buf + data), '' if not data else binascii.hexlify(self.buf + data).decode())
|
||||
self.close()
|
||||
return
|
||||
self.loop.create_task(self.handle_request(request))
|
||||
|
||||
|
||||
class BlobServer:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str,
|
||||
idle_timeout: float = 30.0, transfer_timeout: float = 60.0):
|
||||
self.loop = loop
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.lbrycrd_address = lbrycrd_address
|
||||
self.idle_timeout = idle_timeout
|
||||
self.transfer_timeout = transfer_timeout
|
||||
self.server_protocol_class = BlobServerProtocol
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
raise Exception("already running")
|
||||
|
||||
async def _start_server():
|
||||
server = await self.loop.create_server(
|
||||
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address,
|
||||
self.idle_timeout, self.transfer_timeout),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
log.info("Blob server listening on TCP %s:%i", interface, port)
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
|
||||
self.server_task = self.loop.create_task(_start_server())
|
||||
|
||||
def stop_server(self):
|
||||
if self.server_task:
|
||||
self.server_task.cancel()
|
||||
self.server_task = None
|
||||
log.info("Stopped blob server")
|
||||
@@ -1,4 +0,0 @@
|
||||
from .ledger import Ledger, RegTestLedger, TestNetLedger, ledger_class_from_name
|
||||
from .transaction import Transaction, Output, Input
|
||||
from .bcd_data_stream import BCDataStream
|
||||
from .dewies import dewies_to_lbc, lbc_to_dewies, dict_values_to_lbc
|
||||
@@ -1,125 +0,0 @@
|
||||
import struct
|
||||
from io import BytesIO
|
||||
|
||||
|
||||
class BCDataStream:
|
||||
|
||||
def __init__(self, data=None, fp=None):
|
||||
self.data = fp or BytesIO(data)
|
||||
|
||||
def tell(self):
|
||||
return self.data.tell()
|
||||
|
||||
def reset(self):
|
||||
self.data.seek(0)
|
||||
|
||||
def get_bytes(self):
|
||||
return self.data.getvalue()
|
||||
|
||||
def read(self, size):
|
||||
return self.data.read(size)
|
||||
|
||||
def write(self, data):
|
||||
self.data.write(data)
|
||||
|
||||
def write_many(self, many):
|
||||
self.data.writelines(many)
|
||||
|
||||
def read_string(self):
|
||||
return self.read(self.read_compact_size())
|
||||
|
||||
def write_string(self, s):
|
||||
self.write_compact_size(len(s))
|
||||
self.write(s)
|
||||
|
||||
def read_compact_size(self):
|
||||
size = self.read_uint8()
|
||||
if size < 253:
|
||||
return size
|
||||
if size == 253:
|
||||
return self.read_uint16()
|
||||
if size == 254:
|
||||
return self.read_uint32()
|
||||
if size == 255:
|
||||
return self.read_uint64()
|
||||
|
||||
def write_compact_size(self, size):
|
||||
if size < 253:
|
||||
self.write_uint8(size)
|
||||
elif size <= 0xFFFF:
|
||||
self.write_uint8(253)
|
||||
self.write_uint16(size)
|
||||
elif size <= 0xFFFFFFFF:
|
||||
self.write_uint8(254)
|
||||
self.write_uint32(size)
|
||||
else:
|
||||
self.write_uint8(255)
|
||||
self.write_uint64(size)
|
||||
|
||||
def read_boolean(self):
|
||||
return self.read_uint8() != 0
|
||||
|
||||
def write_boolean(self, val):
|
||||
return self.write_uint8(1 if val else 0)
|
||||
|
||||
int8 = struct.Struct('b')
|
||||
uint8 = struct.Struct('B')
|
||||
int16 = struct.Struct('<h')
|
||||
uint16 = struct.Struct('<H')
|
||||
int32 = struct.Struct('<i')
|
||||
uint32 = struct.Struct('<I')
|
||||
int64 = struct.Struct('<q')
|
||||
uint64 = struct.Struct('<Q')
|
||||
|
||||
def _read_struct(self, fmt):
|
||||
value = self.read(fmt.size)
|
||||
if value:
|
||||
return fmt.unpack(value)[0]
|
||||
|
||||
def read_int8(self):
|
||||
return self._read_struct(self.int8)
|
||||
|
||||
def read_uint8(self):
|
||||
return self._read_struct(self.uint8)
|
||||
|
||||
def read_int16(self):
|
||||
return self._read_struct(self.int16)
|
||||
|
||||
def read_uint16(self):
|
||||
return self._read_struct(self.uint16)
|
||||
|
||||
def read_int32(self):
|
||||
return self._read_struct(self.int32)
|
||||
|
||||
def read_uint32(self):
|
||||
return self._read_struct(self.uint32)
|
||||
|
||||
def read_int64(self):
|
||||
return self._read_struct(self.int64)
|
||||
|
||||
def read_uint64(self):
|
||||
return self._read_struct(self.uint64)
|
||||
|
||||
def write_int8(self, val):
|
||||
self.write(self.int8.pack(val))
|
||||
|
||||
def write_uint8(self, val):
|
||||
self.write(self.uint8.pack(val))
|
||||
|
||||
def write_int16(self, val):
|
||||
self.write(self.int16.pack(val))
|
||||
|
||||
def write_uint16(self, val):
|
||||
self.write(self.uint16.pack(val))
|
||||
|
||||
def write_int32(self, val):
|
||||
self.write(self.int32.pack(val))
|
||||
|
||||
def write_uint32(self, val):
|
||||
self.write(self.uint32.pack(val))
|
||||
|
||||
def write_int64(self, val):
|
||||
self.write(self.int64.pack(val))
|
||||
|
||||
def write_uint64(self, val):
|
||||
self.write(self.uint64.pack(val))
|
||||
@@ -1,60 +0,0 @@
|
||||
import struct
|
||||
from typing import NamedTuple, List
|
||||
|
||||
from chiabip158 import PyBIP158 # pylint: disable=no-name-in-module
|
||||
|
||||
from lbry.crypto.hash import double_sha256
|
||||
from lbry.blockchain.transaction import Transaction
|
||||
from lbry.blockchain.bcd_data_stream import BCDataStream
|
||||
|
||||
|
||||
ZERO_BLOCK = bytes((0,)*32)
|
||||
|
||||
|
||||
def create_address_filter(address_hashes: List[bytes]) -> bytes:
|
||||
return bytes(PyBIP158([bytearray(a) for a in address_hashes]).GetEncoded())
|
||||
|
||||
|
||||
def get_address_filter(address_filter: bytes) -> PyBIP158:
|
||||
return PyBIP158(bytearray(address_filter))
|
||||
|
||||
|
||||
class Block(NamedTuple):
|
||||
height: int
|
||||
version: int
|
||||
file_number: int
|
||||
block_hash: bytes
|
||||
prev_block_hash: bytes
|
||||
merkle_root: bytes
|
||||
claim_trie_root: bytes
|
||||
timestamp: int
|
||||
bits: int
|
||||
nonce: int
|
||||
txs: List[Transaction]
|
||||
|
||||
@staticmethod
|
||||
def from_data_stream(stream: BCDataStream, height: int, file_number: int):
|
||||
header = stream.data.read(112)
|
||||
version, = struct.unpack('<I', header[:4])
|
||||
timestamp, bits, nonce = struct.unpack('<III', header[100:112])
|
||||
tx_count = stream.read_compact_size()
|
||||
return Block(
|
||||
height=height,
|
||||
version=version,
|
||||
file_number=file_number,
|
||||
block_hash=double_sha256(header),
|
||||
prev_block_hash=header[4:36],
|
||||
merkle_root=header[36:68],
|
||||
claim_trie_root=header[68:100][::-1],
|
||||
timestamp=timestamp,
|
||||
bits=bits,
|
||||
nonce=nonce,
|
||||
txs=[
|
||||
Transaction(height=height, position=i, timestamp=timestamp).deserialize(stream)
|
||||
for i in range(tx_count)
|
||||
]
|
||||
)
|
||||
|
||||
@property
|
||||
def is_first_block(self):
|
||||
return self.prev_block_hash == ZERO_BLOCK
|
||||
@@ -1,743 +0,0 @@
|
||||
HASHES = {
|
||||
0: 'bf3ff54138625c56737509f080e7e7f3c55972f0f80e684f8e25d2ad83bedbe2',
|
||||
1000: '4ec1f9aebc8f7f75d5d05430d1512e38598188d56a8b51510c9e47656c4ffad9',
|
||||
2000: '5d5965e43d187b6f8b35b4be51099d9ec6f7b0446dac778f30040b8371b621a2',
|
||||
3000: 'd429f69a9dd890f7d9827232a348fe5120371ed402baf53c383e860081f6706e',
|
||||
4000: 'd18fef650f23032e4e21299b8b71127f53ff6d2114c3eac4592a71f550dc5071',
|
||||
5000: '12c1dd1b9cda29eb4448e619a4099c7bb6612159e1cd9765fbbabeee50f1daff',
|
||||
6000: 'ebda54cab7979bc60f46cab8d36b6fcea31a44c38d6e59bdcb582054b6885cf8',
|
||||
7000: 'd2be08791053336ae9bd322ee5b97e9920764858f173a9e0e8fb54ee5d8bad0e',
|
||||
8000: '94583639d9e2ce5eac78c08ef662190830263ea13468453b0a8492c3aca562fe',
|
||||
9000: 'a6ebb24a3ec9bb3fbdb9d9525729513dd705fe85093c33a07bbfe34ba1f1eae0',
|
||||
10000: 'a5d32c6cd43097f3725e36dd55d08ec04cbe62e40d1fb3b798167cc9c1fa1ce4',
|
||||
11000: 'b5cf8df354bc26e8b0315e8038564a9a7fc1faa6ad75e67819b7eafbcfdcb145',
|
||||
12000: 'ae7b0e5106299e43b78f337fe15e85c27f88d5bcf618a21c03ce43c094dde909',
|
||||
13000: 'b57dfe631571b654e11fb4295aecfc11f612664d22d64a9690f64c0b3ec128bf',
|
||||
14000: '76e55cc1c38d9a6a349c9a42ae0b748d0d8b27ebb30539be8bb92e71e7abca17',
|
||||
15000: '5d33cbc872acb824a89ef601aa3c918e7f4dbb8d5be509a400fa41b03f7fe103',
|
||||
16000: '25760f6a6bfb58104d02e60efa62ba5ae141bbac1f65d944a667ad66fbef15a9',
|
||||
17000: 'ad37d98bb272aee3a950679cdd4cc203b1c129bb3a03fe74223ff858badfcc50',
|
||||
18000: '9eac34fc453b97b1c62228740efcae4ef910b0bea26c322e17e3474b81713b0d',
|
||||
19000: '06783429c8f50cc6495da29d38c86c3873800cf1eceffedfffe2a73f116a39d3',
|
||||
20000: '0237c620f7f68d9ae9852d00551c98a938352d2e3d9336c536f1cbf5246fade6',
|
||||
21000: 'c4dbb4eea7195a2cf73de2d49b32447cec9c063051acbd300b26399c9242b0ed',
|
||||
22000: 'd18fea80b659d9690830c2f37c7c9d53f79fb919debb27cc2131044df26d7ff5',
|
||||
23000: '0371254f4698d3c46596c1fd861e2e1e75053beb69c3c4d8ee6702f0992f0026',
|
||||
24000: 'dbc97c3986bb9bb6230321d1fc8b7ca13369033f57b08ac01bb775f7241861fe',
|
||||
25000: 'cfb9baa125e91d02c060940b5b51cce52b495a0b0bd081cd869a7ca45a0e0585',
|
||||
26000: '6e07fecd3d0f72b8cd50a3cdc1e9f624beb3dac9f6356308a1ec295f47d68f3b',
|
||||
27000: '8ad248da3aac7c1db958d0ce8a2d50fde461bbb96a628efa215116d3b74b74e5',
|
||||
28000: '4e8231ede23b397c49f1a97020ab5fe45465936cb6c8cf1c39ade1b38a80547f',
|
||||
29000: '1453b7a4be4076e2c77b94bc35f639c571dc05b225a5f8018a9a60337fb3ef66',
|
||||
30000: '43d779c83032d75e8df09aeb6ab103ad41618777565fabc2e56bf31c3a20285a',
|
||||
31000: 'ea1426f57f75ee781a6e569772944119f5e65c8b110b31a32b61d951c3a1900f',
|
||||
32000: '43d1a3941b2c3033f78040830d39e657f74958d71b3cd0bc728550469cdb5fde',
|
||||
33000: '3b0c0eac231476efed2b9bb9dff206a6438a14946c3b4238bf0af6d03f9a350f',
|
||||
34000: 'a8136e9d6f3f5048ef8f9f56628009f5a87bb28664ba8028ecae0797e5328309',
|
||||
35000: 'd8136e638995aaea03c6b50f1cc0f1891bd107290f12a6ee57e3cae36b7e3374',
|
||||
36000: '06a7692cf5ce9bb71461e5427e6f55bce2904f9d3af4c28e7343fecbf0e336da',
|
||||
37000: '12a1a49c237497295150f99c76d79d8affe9fd720e0cbffa62be80e2bda7f832',
|
||||
38000: 'fcf3e526931b7f7cead47652f12cdf76641032bc545e22aa0dc83bef46a085fa',
|
||||
39000: 'ffc34a198ab1cc13819c0c90b26a8455627ff12868b49ac1e6b5e1a086ed011f',
|
||||
40000: 'ac71a9895999531350cf77a701035b0b59720a39994c990354b9cb1f6ea8eb49',
|
||||
41000: '4fa753f5f2de41ad69ab32c6802eeb352f070d8684c3bd154eeb8a3c17aa6363',
|
||||
42000: '6f0febbddf9248e3df2254a7196ec13f353c7a8049e098f2fcdf920580abffa4',
|
||||
43000: 'e3e98488cb203be9b531e3b5ce2200f4e5655297a4f4534bcf486ad5e26687bf',
|
||||
44000: 'c7c6078e30163204e49d493d506f72f71d68d2a591874328342ca3cca81c87bd',
|
||||
45000: '2a65629529f69c15a6e404ba54b83e024acba9d6c786be554fb14d28951ddf9b',
|
||||
46000: 'cbc148ad661e6fc390746e94cf21df71edcb222b4e8d8f0771237a8e03412f9c',
|
||||
47000: 'd6977f770749b7ce1a624a61f7b2d26d817a27c50a895466ee10f8bd758c029a',
|
||||
48000: 'df05fddcd5ccfab7811e5d7b6cf7b49df71c7e0f77a57ce384936948114bb93c',
|
||||
49000: '9d26a67c06f066229bcdafd55cbe71e2b2518465d6deb618c418b2f229ef513f',
|
||||
50000: '4ceb896d901315e0f3e10f3b1ed8472f5bb5230d4ed6377214b07e425e515f6a',
|
||||
51000: '218019bd8440a8c242e84940119d4eccccd26134a208cbb92cdea3480069f482',
|
||||
52000: 'c15f1538280e71abc1cf3e117c79b0b671a9f6fb46102fea7b9a1fbcad418011',
|
||||
53000: '0ad4d3812a42e23fc0defe7c62dad451266ee21699ade625a5f00b91993850d3',
|
||||
54000: 'f994cff7b8166b0c39a310bb686f043fe9355678ae406a7fa1a74d4bf6f9b849',
|
||||
55000: '6723d30c99fb45513a42a3ffaa50d1b8350e3c8b52c6dc6894303147eba95d55',
|
||||
56000: '947cbea79aabe2051522d9e957d92b86dee2c420c32a83cf6f6d655136731926',
|
||||
57000: '8a153fa31a971e7e304650257bfdbaac230b36a9beebc01a85a7aab9a357a5f2',
|
||||
58000: 'd19a5234ab995cb851d691b2c2c84ae665890621c52794e54c16d6a87f516c7b',
|
||||
59000: 'f9c7a353e62ce57a5e3f983362b08fdbff5e2aaadce325bba7680104a6f51cb7',
|
||||
60000: '9e1d535bd31525803e93e4b4c2346fcdfee7c5cdc4b9fe3032df9295aa85c754',
|
||||
61000: '640387271e7eb0816b9e290dd157f58d1eb54cfc39bc7ecc4486099af2a22b76',
|
||||
62000: '3d36acc4169a238657729df674277c6cf236e21c2bed4e99b8f1a6e82d00bbd6',
|
||||
63000: '6d4201e45595ea160773d63bb1481f78664156de1f4564d9a8c3f61b3cc1a6c9',
|
||||
64000: 'b2cadf2ba39b2d5e688029cbbc68cc6d71337b29861dee24fe941b31be1198a3',
|
||||
65000: 'd8a364d385ef42e53ce88c82355235ee7a27dd3e642849dc22d08a4041695dfb',
|
||||
66000: '7025d4b2d9537002ce2b9ec42c596eb864e30f41c06f02413997b5629d6dce3b',
|
||||
67000: '852a3fecfb3a77bcd6d2604170cbe9646d9b889c90d234eddc8f9d112777fe41',
|
||||
68000: '38956213012f66a89377f4914db838faa6738732f7a7a865ebc5777f2f126b73',
|
||||
69000: '4c89b07d23727d4bd1a080bcb9a22dcdb45c904f4c9866153a000056d7ec1546',
|
||||
70000: 'e4e7fbf0d17b5f933deba12e045ea23eabfa6f7dbdc633e5c3d3ef8ddd066b47',
|
||||
71000: '1f07c36e82f8e7582ec9df23ddc90715df45978b6aa0a2f17c08ee14cbb23000',
|
||||
72000: 'c59c660b5aca2fa3364524ef43a4fa93fc3f16df777427cf1d810f65ce42e4d2',
|
||||
73000: '83a05ac2fcae8c5d50a90df308f641616fa9b9fb553763c874b764a73def2f95',
|
||||
74000: '448d209c032a2c6f5330481bf34fcf0bf1ea2c47f43fa5f82cf42e0d8f4d2b20',
|
||||
75000: '5627eb3491e1154ad2c90094da85dc7dbfaf89d3765848db1ed915b7e26dde58',
|
||||
76000: 'b1417b1b2360e24fe42345508ba0a333f4bd76f48c959b8bb40a99bf34ec5ba0',
|
||||
77000: '250aca2bc4e6da7a8e1cb2e4deb0396aee7c317177b099d7dab9731b4c0f7573',
|
||||
78000: 'dbfb1dbe80a5db06c9dcc645c5279eaf40559f5e04df0e64b696e80aba5659e0',
|
||||
79000: 'a7183a545925164b5a9882810ae117d0f3df9b3ed55885fe74ebea284bec484f',
|
||||
80000: '5b4a7bfd9b5394e2759daecfe472ea6e92b281a7821b106314b2b2e0facbaab8',
|
||||
81000: 'b7075fe530b44e4f5895521b3cf99a79fe23d8bedde0027d6e7924ca93793eab',
|
||||
82000: '6a1fa75b896d799ae84046cba7cd942dad47ec04bf50a85cd1b2bb15861995e1',
|
||||
83000: 'aa5f4caf970433e90e2e45f9524ab9a6e22281505ada4c206ecf4672486240e3',
|
||||
84000: '103732d12ef792a1adbe8f295ca6abd008867323ee996698b2650bdc1bb8d06b',
|
||||
85000: 'a214c16ffdc505d6caba7bd0c2bae766bb19f7eb4d436cfc037e93783afae7e8',
|
||||
86000: 'f54226ae9a6814a345968b5c2982adf638b995ff50e27de1890b3760ead12158',
|
||||
87000: '26546212a0720cab988194432f4fa7c3e47caf0fb8e31efdd1ee93c3ad056868',
|
||||
88000: '48f7d47bc443bdbc6a37ffe8f1d0d91de16d4a470d03655692de8a04f84c2561',
|
||||
89000: '005bac455ff093a39b907af0441897e7a9ceb19eaf51d9dd9ec2b5ea43ff6d57',
|
||||
90000: 'b482f10315170e6bd280325e180ce37b0423b685baa06d33586ee659649b71a7',
|
||||
91000: '56db4d5d6c460fa76d492d9900c02e8fa99bfe97c7d9fa2a75bf115128ea0da7',
|
||||
92000: '72ac917c4f531f91f65fbe33e78b4d4e2bef18aab3d2ff584d59fed1ba2cf398',
|
||||
93000: '1100df2fdb03d44530723d131dd7d687053db6a317ac9181cfdd51925152df02',
|
||||
94000: '6d50f66efb571136501340fc00b52105e0011f0f11fd68c9b185717bd42f9307',
|
||||
95000: 'bd0c996d9ec40d5e91c2b5f278a5c9c9ea0b839012a7d135e1afc73a725bd8d6',
|
||||
96000: '4ebfe8a3e1a1625632896d5d2970d8b080f08c5948f0ac38e81f3fc85db5af5e',
|
||||
97000: 'cbc03464aa3513aa33b540d0f026b93db7607c4bc245af8fd88c3b476ea5e394',
|
||||
98000: 'c9a185ffa55d01fc73c2e60004b0c8dee60ffbac7784cd16046effd7a0a51a86',
|
||||
99000: '1eba97d5ee69229f4cd697c934d60cf3262ebcdf4c79526cda71e07b52fa22f7',
|
||||
100000: '0b1650207a55c64e7a6cec62ddc3cc190ed3796507bbd48358a0b7d6186bc3ce',
|
||||
101000: '1f47841b0034d3cf1046660c2246902f679445b5ef621df2762bbea33b7d685a',
|
||||
102000: '42f63b1b622d08e120b5405ba65d25e1a9777312cca77248b7827d7084e6d482',
|
||||
103000: 'ef4861fbd4e578ff80e0d5e2afc62fb113aa203d3933f74efd3e2400d73de922',
|
||||
104000: 'e97a3cb78c09eef732757e81c6a7135beb33d95398054b45df738853acedaaad',
|
||||
105000: 'd12e8feec15893ccab662a1ad0754b2ccbc18b078ff3894eb2214ee4ac2f7af9',
|
||||
106000: '83ae84103b37c93b2d1535fc47b053194270b3c186b1b25d69cd8a1540caaa1a',
|
||||
107000: 'a102c6dce88bc4695250c24a0ebd2913b2b85c211b7d5dbbd18ecd95bea63144',
|
||||
108000: 'a0c20b36140d55288dc4dfeb6c16a6ece3d43efb57b1728bc872c35d6660c704',
|
||||
109000: '412fe428b2929c3cf5c6991a2657e5414a569b30e77a3f4fecef14239d89caca',
|
||||
110000: '8518706e957f3ef892d1d2d7f65c5f4588620be994cda9cb7d81fb600554a456',
|
||||
111000: '66f4e25f40d24360299e5358f954d8a79b245399c46dbc4e8a3db0c10fa14e18',
|
||||
112000: '81c0ded5c5e30f92eec84ca293abb0165068d917bdcf436754a0e9b0ef1e325b',
|
||||
113000: 'ebd5e034a45517e59e96e986b55e38051a5d2a3120a7ed92d4f3e01e18e73972',
|
||||
114000: '681b85720d71ab660c7443392883ed4bdcf9aceffe202cc5ead94664e315a744',
|
||||
115000: '3fbda839115bc6e6d451a05f78b27b35b137c94374fdaa42235f3035980e85e5',
|
||||
116000: '7bfb88d39fe0ee7ce046648676b226997a812af1f1ed79040750045536c76067',
|
||||
117000: '5853c6d48fbf99b06d3e1f474e1b667ca598712f9feaff8cda48a95935cfc498',
|
||||
118000: '6a23987731379799289d2a527dd40fde8aaca8398625d2a2e3e646372ffc99e1',
|
||||
119000: 'af73cbe957865b60c3c3139dd1e2bf8bf1cc504dd167545463105996609750c4',
|
||||
120000: 'b309a20170421fc4ddfedfd7611716e1cf5a802a1db09b66b9ae448fdf958792',
|
||||
121000: '624cc893801e4da093ca973e2135352bb428dc278e47f5308412d05acbbdeacc',
|
||||
122000: '2f20c67ac3485b6e5407762a23e8a7df1bb380f1607f538cb5e2e1ffaad578ef',
|
||||
123000: '89c73ed65ab8f729cdcd678dc52e876969907f1560898220702a696893744e00',
|
||||
124000: '17d8155d97fa4e8705aa55969709f9cbc97f87815d40f5a2b5cf4d7b85dfe4b3',
|
||||
125000: 'd3cbbdb7f51c8a1f5794bb3d1f40c413b313f91bd3f3dec0d8ebb6a02107e52a',
|
||||
126000: 'bfd02d760eb5de2e792ea74215ae37d13174c3fc0aff08587281f0f39e427caf',
|
||||
127000: '937dcc09a14d1de79f5eb679996225995cdce69497cff5f12c39c098c42656a0',
|
||||
128000: '81d372b07e2467bfa62bb4b974d0eb75ea313e547b9d4e62f3c617f939bcf67d',
|
||||
129000: '9b4b315738ab8b5266b5accada140dcf487b2f509f1d758b9b972b93ecd03c9e',
|
||||
130000: '8ab3b1862cacf7f1c4f6617f7280ba31547dadd62bec9c37ecc8d074c90be2b0',
|
||||
131000: '868d6520e526b8594c44a05eba67ec1de8e01b547b34b62ebc5dd09d656e9072',
|
||||
132000: 'ce09471822965da31db3188cd156538c44337da2952486a34dd2ab372ea490a8',
|
||||
133000: 'b0c34cd700630c9281c8657dd24baf609f72f897442538d77f4ac355199731c0',
|
||||
134000: 'a76e5dc895406a3951d55b557272915810420bdc3ac076863c7efdef3b115890',
|
||||
135000: '2b70fd7021d40f6df15f3f72bc8e034a8a0bcd8542847b5b7a2088253351b3a4',
|
||||
136000: '6161ee84f6fe100a02de0675bc9500dc18ccaec74e7da4b92135f0ccee6fb663',
|
||||
137000: 'ae5770a7158089ca9adab8bdd07f1259e6f204ed377af27fb6a772fe1c031864',
|
||||
138000: 'f3b8de21370968a90cedfcb1cb6803ec9a6b7740a94d8e4c60a80407fd55a12e',
|
||||
139000: 'f08f6431d7367b71b25b321809ebf48f797bf7fcc943ba366420fd3f3dc00e5b',
|
||||
140000: 'e76345dbb8c4c5ed1ea37249f892f64098557700259fb360401a559c45909041',
|
||||
141000: '776b333f5b221f6b443d6011bc8d0753eed5ce2446a6d3ee99a5800c7159fa92',
|
||||
142000: '9453693cb846f27ba2ac39194e5b70cedfaf435b73b5f7f288dd81343ff605ea',
|
||||
143000: '08aea64cc4eb0170d2704cbd3a4ee871d4f3e53a8c1395288c029eaddd1aa097',
|
||||
144000: '801d60c225301481c761114d5152780d5002b77aed18845c32c9e3804a5db254',
|
||||
145000: '78cc80d54933b84248aee32a18144d03237f948de7889c97e2112d17ac998009',
|
||||
146000: '1e954e8f2fe59da3feef0514f6b271f52961467782b2530501c294a83181f176',
|
||||
147000: 'c0ac42fe9c0e1d2b0b7e6e796c553eafbd5d7bbce7df84903dabe16dc56ba863',
|
||||
148000: '75b5441fc1785dcba69b9409f24224ba3207d624bb0e7d54af24118d08c788d7',
|
||||
149000: '20e53048515cc34507cf2d1dd160eabafaa97793d81626e9fce38d540142dbc4',
|
||||
150000: '5ff7a08461fc3fbacf8c6261dbd34a790e271bbc39908c23d5417f0f7a71b71c',
|
||||
151000: '5c14cab7670821966d5aa61f09b32d93c865764b6e258764a3c33a82a3133fe7',
|
||||
152000: 'e595968e040b54ed6572862b4089e005238399014d99bf4c361cc5922dbfa6c0',
|
||||
153000: '1292bac3fab11de09f46a8327511b7c37366d0f2a96a027f4bf2f16147a39af3',
|
||||
154000: '3db31267722b500dd9b26f5cfc4f06f26c5c343d181464499d3bb4aaf13c3c20',
|
||||
155000: 'b92962af73b42335125f28d7d3fb97e3aa6ccf47c3f4229eb7fa04b67fdb933e',
|
||||
156000: 'fdbcf12415284d8cc0adbf16996b9d37a943fd1a924c5dee5f47dc8add23f278',
|
||||
157000: '199781ca8e5929708f834d6f38ce3e2fe196ab519f9915f14401462273431e19',
|
||||
158000: '969017fd15cf6a73683de287c5e19fbd69ebb5aaf7c13339297f7177a716de3d',
|
||||
159000: '77c7995cf97218655c195fdac9010c599859a46f760d84750189114d2d2d1d9d',
|
||||
160000: '00ff84f0d845b4f1099d970cc0c3e2dd1c2584c611449f2318a3f27327246d51',
|
||||
161000: '08481dc5f61e776ae8be12236d595549abfa0be28b187d80dad573594d94c11f',
|
||||
162000: '71497ab26b05453f3c7057f8bf57fcc8ba30920a6032ab0ae3abdd29e0677582',
|
||||
163000: '08713e0b750233a3843241d24573c4800f32c894000b7815860048ef3e7a06db',
|
||||
164000: '01fd806f1879285ad5234f38789074b0ea3a0b2707bc6b49aa9bd51ecd26ddcb',
|
||||
165000: 'a990225a2f02a77c1c61d518d46dde1e1cff3f4df0ed24eb463b97f84c7a2616',
|
||||
166000: '7772a601a6e765ee340eb666cd645503c9988430ba4f558961246e9e69349b25',
|
||||
167000: '389c7a8979078b57d54936497680c7908b9f989e46631358f1c31a3094621557',
|
||||
168000: '2e1df97fe3cba7add5f05734a119464ff7c22bf92701ed85642da856ec653aff',
|
||||
169000: 'fe0adfe455f65cf90e1632ef39bf9ca5856020d995cb71ce8a4998a40eed5998',
|
||||
170000: '87487f255873e9f6411565553fa9ccb9518e7baf71ade67492536278f1ba0feb',
|
||||
171000: '37cf239bd630b7c891019adcefead4baf19a86b40e21f3dfe4a76a78f2985103',
|
||||
172000: 'e2f4087b868558af51dc8f40d77c300d76eacab17e5fb8bbf3b1f99c02d995a7',
|
||||
173000: '273f32717f740438e93063d259dd4f0f160077c34c937a7d6a9b1f9fbb34d87d',
|
||||
174000: '807a861813ca690a6fc0715f354f36bf491c34588b5ab778c0f4e692ec3fc152',
|
||||
175000: 'dd74cf3d686c59820885bab134c81b7079f2edca86a25944fe0aa36a96941550',
|
||||
176000: 'aba8163b91a905aab5ad6e7599b919fba650c446938dc7b2352759203bea8957',
|
||||
177000: '260bd0d053038a54565f9b25c3894cfa875c9426b9d6e9915ec1df03bcf90ac5',
|
||||
178000: '7a527f71743c4dff25f13b918ad4a6d91fb0bf9c873a7282e51cb4edf545edbb',
|
||||
179000: '945397818b80995ac4e23c785b6d5ed2eb01338a2d4ca6f0cf40986e87659d86',
|
||||
180000: 'ba1b5263f5418844796a19db06a53c095ec0428696d8ee953790d6c86de0795a',
|
||||
181000: '2b673859af43554accd8599bca53c2ca94500ecd65df7672a1e586595bdec7ff',
|
||||
182000: '08bae9f096e348c8c6b42ac2762298ec3faeba896524968478fff1d40017b5b0',
|
||||
183000: '211577ae9a4f93fa5dc3e33764c31048ee306b5e5ffa5081fabdf3f30e49a977',
|
||||
184000: 'a4301eead637043a8751a30c13935af971a731e03e90317c5a16c1677e50d837',
|
||||
185000: '1f4aa362281293e8328b6c5b32a9948ef03dcea81c6ba13f94531f0c5e42627a',
|
||||
186000: '0c6b15f2164a1b55bdf0eee2b0f5ab7cb9d2dfea6fcc82686483aa7a3659f6a4',
|
||||
187000: '5bfa261cad0e4d271814de1f4752352f35da10f24b8016ac0065172e1296052a',
|
||||
188000: '94c2490bdafe70d65951e4034e5b9122fa6b376a5ad8c9e6276289c4a5e059e0',
|
||||
189000: 'a0119c5d57523b7ccaf1e5c6671eae9f8562bbf2477f7dd8c6847c6da41707f6',
|
||||
190000: '4d82802c2464053c391374eb6124f8839faa9b343bae3f34e00f87fad9f45f9b',
|
||||
191000: 'd341f5b4d20894be520eeb533d56c5737ad02a7ab0c24b2463516ff04e3b8b0d',
|
||||
192000: 'ddf1a72ba512786d39f7c313f710bbd452040f1eb8a0c0ee7a4b9192199fcc3c',
|
||||
193000: '514e1b40e5e15dc72e75cabb8d490c5e76b968e6dcdb3acf518b47e3a2020407',
|
||||
194000: 'd0bac7ca0636b662d6be169bb5af807c891523a44fceda68f08c8934d6e9e254',
|
||||
195000: '00c37e81f0dfe63a8fdefd0ba91442019899d50f72b43c4de8017e767777a5af',
|
||||
196000: '581a9d093458c0157464c42fa2e2d9a1f61f7edf1ccc5dd77a4fdfdd9a4b37b8',
|
||||
197000: 'd7555fbf7671d2475e38f62418f6130ddfe85b8fd07fe8a9815849316165d401',
|
||||
198000: 'ea388884ad5abb06ccb1d0d2202dfef5c4ce4de0f3040df089017d01f14b9530',
|
||||
199000: '6a80cbb22620b7a5f99f4ea36380b7bcd22e87feaf4186bfba0b9aabcbcaab70',
|
||||
200000: '7a7231293d242887c6dfa517a78b60d40b41456b6f2fe833c0217074664a61d7',
|
||||
201000: '0d6bde8cdd0370964b83ee766891b3cda7b9eec098dfaa5ef05842b29bbd043c',
|
||||
202000: '223fc091bddfbaedec92099f873fc1a6bc50556f3421f472a63977b3351b95c1',
|
||||
203000: 'f87fb207fd397f8fb5c5c21f84f272cf33d840e20ccaf9de810af784d8459142',
|
||||
204000: '6273b130d819d84492bc50fa443ca401df748e58fa84a3708a1797f116f60f1f',
|
||||
205000: '498edfaaa9d5f0501aea5482ffdd9208b86af46fcea5a1cbc82d51d45eb6a256',
|
||||
206000: 'c069138610493a70742918b5c624cbf9275bf55ee24649e32244238433883ea0',
|
||||
207000: '5110744882ba11b98df95f11e100bc398a5aab4c8258133b518b5abb58c7fe64',
|
||||
208000: 'e5690d4e0561ef015d6e2997df3b8b95163cd884d999d8c9ac212bd7e1f06ab0',
|
||||
209000: '17a0a3bbf2fecbfa5c7667524cb2a452f49b8dd089af132f9d2c530ca0b677b1',
|
||||
210000: '3a9489f3aca89576152949a67df6462d25bf3a78976451b5cf8bd8385a10e6ee',
|
||||
211000: '45196e756c7c282c4a9e7133adb07f103d0f74ee90919cc8ad89ca3e5d38c6db',
|
||||
212000: '590f21c83694c35028b9c9d632a082307b8bb27ec87e2c0c4e2881a7be9c6637',
|
||||
213000: '46e56c45305c51e4d22927e449a1855064504cfb58cfceb0f293d8a4f1dca7cb',
|
||||
214000: '50b3f1bb82c1b213ed2cfd2c1144b8e67e5fa731dafabfefe063f8658c5536d7',
|
||||
215000: 'f53709db4ca6c12b80c64501eeb8f176211116b0fbc3a650082cf830792b206a',
|
||||
216000: '6818db3f6582e71021a84fffe751fcb47db2f734d59547be47a373575b5c7d8e',
|
||||
217000: '01f6f8a0699ac533ee7921f63334af07b04369d5e580c1c609c6fee64c8c6b8d',
|
||||
218000: '81e9ff93cdf5686fc42663a056c74fb0f560c29a5d23f21ef92364735a865392',
|
||||
219000: '80b32bb6bc3c54a19d3b604f56084b3bda13afd4af4130f791505e44fdad6c98',
|
||||
220000: 'c427f24ab14449b1430c5302385f3a77f90f19715d940e8dfb7c054828270163',
|
||||
221000: '4303a0e1c6f8489589290370a6843d97eb02a01be8c227d905cd0e30034feae6',
|
||||
222000: 'ccfdc1768afc18cccdf2b5d90d40158d86b110d878955b6570ad9e6e40063008',
|
||||
223000: '6baefb7061239c02472c4783de6d32936f7fa1da9fa6d2448b4d92f40b89e79d',
|
||||
224000: 'bdc334f5d1a3bd1d19cf589e3b1ab0e8a324d44ea1c70297ff2de6a66c99399f',
|
||||
225000: '82d29612f96c55dbe83655db68afe39146aff1418af929f05033fcbe612ebcb0',
|
||||
226000: '179b4144f4d644f546f001273c1e5382a8de50f84ec06a2c83b02808a9167d28',
|
||||
227000: 'a350e96895efe979223475151305eb8d1739121b566964033b9967031ee5581c',
|
||||
228000: 'd208c66a9ccb1afde8d1540d944bfb43522015f9d2df50851e66ce1b84b06c0d',
|
||||
229000: '95cfabe5d18dd8d8e4a57b7bf7052ab09b7138ba0b3c9d2efdb32352bfcca149',
|
||||
230000: '3e399d08c33b8fddeb687deb276afd3ce101d19f5a71b232532951f05d46d8f6',
|
||||
231000: '91c0c17d5df1c553bd7dd6db8feb58a4436d598ef4acb1ed9a06da65f3cca83f',
|
||||
232000: 'eb3928a522dddde176cd4c68fa95f1e953d74f3759af4b030f7e2be4c466c45f',
|
||||
233000: '3ac946df5a319d31716f50c752a5769471c0f3ba4b1002613996981c8f32fa3b',
|
||||
234000: '4c9cdbce1f2a4324368130c154e2d24e885877c13a126a847059ccc289fc922c',
|
||||
235000: '8701d98b254fd0b588f022ea140e936d496ca8ccc0890ce133523432f4c09e6a',
|
||||
236000: '0d2d45adcdf950541c25ff5b2b87dff5faad479094b1f68804a62c3151e6d598',
|
||||
237000: 'b8924ead720d0e7a304c45c209ef3fdb90cde437d138bfed2f1e8357fb3d951d',
|
||||
238000: '4c9c62a3b3f6bbfb08edc23d63f9b35ed94bb38393fd1761f9b9b810bb72f68d',
|
||||
239000: 'd01494b4999c2c4def2564eeb1207fd1f3d35a62fba2e61b386bfab498cb9b6f',
|
||||
240000: '3021d8f0edbc0cd2ab87bad5feffac54924b51eac9f6fae8ebe289f474f11a8c',
|
||||
241000: 'd2ecf0170093d3a2dfc876f673e565e3e4aa76d23896af51c96c004cb14ce67a',
|
||||
242000: '2b5ef67326178702a1a35fe4a848d22b791114b59121e41a25ddc1bfac82c6bd',
|
||||
243000: 'b2d546f127252c3fa5f7ca45c61268dbaef66027484a1bcdadd281fabf97d34f',
|
||||
244000: '58f9b398b871dd62952899d31661c118ca4f0d1c71dd165523b889d62b154393',
|
||||
245000: '504760f0e46e2de0a2c8b4dd3e93c19d0f3e0e1b36916f02cbff166655816206',
|
||||
246000: '15ffb23b844e6ac648a7fd099be8e631cb51b3a2496c67ac2dbedc4f6818d7a7',
|
||||
247000: 'e3e56e8575552202cf35fc883bfcfd92cacf8013a6536729ec19e53bdb0db128',
|
||||
248000: '22f75d137c7fe42a80ee04a386fdea8cad56e7ce8601234b6ec676498edf9e34',
|
||||
249000: 'f84709269c393cae8106ddb1fa3a48cf3226e97bcc108717e483493159921aa1',
|
||||
250000: 'f3aee545330f18ec49f6f880d7f9fd31ed6633279881dd13177b0a687c51b000',
|
||||
251000: 'c2e6a4cb3c6e88f523db2ac24fc7fff3b8a27d13ea7b5d614806ed96247025e4',
|
||||
252000: 'efba2c1f91a13ee8d7a8658e70893ccd3014a7019c225e356c3ee67955b58756',
|
||||
253000: '9097ecbbc1279bfa2c92822a6485714ca7ba35cd3dad9ccaff5a8d24645eae92',
|
||||
254000: '00a41b720ec0d7a9fea3229b16ad8d065cd6bb1d14587eb3404de33e45db9454',
|
||||
255000: '7523636406176d391928c638e15e959a9d0255aa71dc9f3bea8995cae02d1d51',
|
||||
256000: '8d3f7c5ded24e2488e25d6030d78d351354734b8b058caf6143430e96ebd5d0c',
|
||||
257000: '48fbbc3737a718163a501e52964eb58aaf4163aab7ceaa4f75f928b25c376ec2',
|
||||
258000: 'e1052e7ebf83246770a740c0c6ce8f011fc7007448777c533d67fec810fe5e64',
|
||||
259000: '043f7d98b500fafc248549c1f5a9727fb0cec4e3e5f13071eeba459fb9179d13',
|
||||
260000: 'ed2f3960b077651ac554b277dd271e12e6cd05942fb54b9464f18432adbe5ca2',
|
||||
261000: '234f17bf2054ad4d3efeae165c3359a14e08d2e53506c839cbed6ba3d2a48ab5',
|
||||
262000: '4342d662c4dab581bf1c123e057e4413bbf74b7a2a331e80f7099201539be54c',
|
||||
263000: 'a7b1380371fbfe64bfb94bed5ae256f7db59a4661e461221cae99dcd54c72a8a',
|
||||
264000: '4f2a606cdc5f346e0855d962cbef7c2e4993b2b07b86ce658a796b1baf6179d8',
|
||||
265000: 'cc09cde8c101eb17b5457bf976525940904cb839e70ef3335780eb0261976826',
|
||||
266000: '3c5f24533723afb124b2bc8c2f8d373eb5640dc4ea4dc76b841e81706dd6b29d',
|
||||
267000: '467c5ca63621be011a07129d0c3ea785e2a0853cc3c0d1e9cace122831d42507',
|
||||
268000: '07df13a794f6c0fc77142fbd0afa23121ee2a2a23bc9abca35c164edb5f6ad1c',
|
||||
269000: 'e2be277e83a997a111fb45948e9a7afdd7925feb1708a84c62968cf6a42da1c1',
|
||||
270000: '463425dddeeac2f27ee717364073229c333a7ba357888623c3b15a1db52542f4',
|
||||
271000: 'd62e0bd8584fae7e4893c18252ae8d1c245e2e4c8e2f2d73703c681c6259f19f',
|
||||
272000: '85cabf5254b6b6292c105e68d44f78d6effeec582467caf9d30a5deeb43b6a6a',
|
||||
273000: 'f5509cee847a757c84adea8e40f3c7c0cc6a4da33a8cd3019cf9ca850d0bc93b',
|
||||
274000: '2cfdb3477de2fe1c81c7133094094c79e933cfe6034c158de07ffd0c4c08b564',
|
||||
275000: '1c1b8e447e95ec3c306fc68528631682bd85e1c876d86223fea28b7ff383c16a',
|
||||
276000: 'b37e5e4f6d29590c9a58773a4d4384cf7c51d04fede4a602ba8095e14bf65993',
|
||||
277000: 'b92d6ffdecdca014c946c2fe8c6c404ccf60d8401e7631918da8885bd360daca',
|
||||
278000: 'c1f82e49c8f66aa580a216c8e08262080ff9fc4f4df1c482ce8b2695b90b7240',
|
||||
279000: '41c48f5965e8939d1df69127798b46af17646ea75e545a7b47049596d9bfb3a2',
|
||||
280000: '48da55fb8ce259432417ffacba5e064df4df2a8293cdd484ee0ad8ce4b0bc38f',
|
||||
281000: '0e892fd450a8ca018567e1a6e23bcdc5b4b872686f816a415991f59b3abd76f3',
|
||||
282000: 'b90fb78edf920d0bfb4bce1f39327c6777b511fd05d715729d7c4cdbf449d59e',
|
||||
283000: '80965fd3817b9509ddbe6a919aa73adb11af64cda5d8352ad2bd9b7824e95c42',
|
||||
284000: '43447043aea93b162d6f615e8cb7d992d02c176a41b0518256f7e0d8fb3162b1',
|
||||
285000: '33bc92c77ba88251822698a7bd02860ef9c12420e4c27f4020370acaab0a2f37',
|
||||
286000: 'eb664123736754c19a47e2afbbc8645c598d7b70c3d42132c334cfeea8fd6bf3',
|
||||
287000: '6af8f2fb3ea1f0fd0d6432599c3a681be7d384cdc553903bcaac8de2d472725f',
|
||||
288000: 'b5fcfef1b5f6373bfc9322164160fe98fc7fb4f3117bf91c8fe61324db62723a',
|
||||
289000: 'f6f1c6f8a2a2905b2ca18845b8bc771bad6e9cc32fd2c8e20a565cbd7781bac9',
|
||||
290000: 'cded625fb7b923d7471ee28276b6983bb4fb4f6281ded9a8637b6974710feab5',
|
||||
291000: '49af69a4855a36d00cfe2408f7d090fb5001913aa06615eca5af78ecb92b1fb9',
|
||||
292000: '2dfe91f7f1870539e6b51ce58f9cd6415ea8fd50dad376d74941a9ff1bf0f8f4',
|
||||
293000: 'b84f14b784cf6eca78d13f4b0db4714be5bcfa59b73974b7106a0206a8993767',
|
||||
294000: '07e95fc3c141dc64ae192cd233f97ac16b1adc9dfbbd685475bf693930b6f604',
|
||||
295000: 'c39cca37abe0b8839f8e797287385c73031496c632a064a89b9dc0730a3fffd2',
|
||||
296000: 'e69afa519ac761328358f5b3ac8b4395d5aa4b955bf1b4c5f3bc0af49cbc3756',
|
||||
297000: '20c93e2d1d9bce1c861eb16febe50e534acc3e2d142cd2d0767bfe076545d1f8',
|
||||
298000: '75970b654ac884cbd1328b68b59963bfa5babacca4c7d48368a1d938feb8af6c',
|
||||
299000: '05448c7f1230870e814858ac9a7858fc72a045dd9e0f1f14b8862d2fdd524b07',
|
||||
300000: 'e254223f85c3ba65a0295a5538387c08ae94e6070f5c989d27349978fef077c7',
|
||||
301000: '4a8bd2357b4ce05422d51e052018fb9f75ce3c07c53b5d1623ac19afff388e63',
|
||||
302000: '18f4a960b810034f5ea02458bbcb9bf3d38d681555be4e40d132f867444306c8',
|
||||
303000: '667cd927ba1e7ab162c17a711bb0ba65f838e543f46b093c86c14e39988da156',
|
||||
304000: '55280e92a3c3b9ab9e1032490aff10c1a2fe9e96cf4f88e49b7a756c0491701e',
|
||||
305000: '8af5b503f1819b23da7b3663314a15b10e39e072377d1b08518a373c85515c1a',
|
||||
306000: '439796b2b4add7dd0fbe14bb3d04f3917e92de2afa07da2a384b389ad9228888',
|
||||
307000: '07fd4cbacddb504e23880dc454fb70659386afa21dab3e862cbe7688218a43fb',
|
||||
308000: '2734a4205be5f2c8ce5850bee8d4ad0829c208911bf4ce833349a43af2db8fe1',
|
||||
309000: '261b84349406a913a4cb700fd218112920910c1c57922e289114c7b903fe2d9f',
|
||||
310000: 'bc652e239edd3476df7bd1e0563595145fd4f2b37dc2ac3bc28ec089a7550812',
|
||||
311000: 'fd7c1b32015a2418efc6e1a73470ea3205de05662c767b0d4198fcaf804c3b5a',
|
||||
312000: '4f017d27b81da7eaf62760cb7fe6613a6ec28fdc18859b6e727b216240a86315',
|
||||
313000: '2d09525e2364e7cd7fe45191fee11f1ce16a2fbd16b56e41f90a3c64ce2f944e',
|
||||
314000: 'c05770490a47280c5b318709c264b2ab951a5bf8c4575cbf1b1e3dbf35323f45',
|
||||
315000: 'ac5da2f5abcee0f92f29479c6355ddafd1f3b4b3cca6659117c17c9fc10fc743',
|
||||
316000: '4496d1a0e4c22a2521595bd4fae60abdb9404f79135311606a9e6713be373271',
|
||||
317000: '40ba7c87490677af9e8b033a19ee5ffe51ba192f5912f491738bb6485cfb76d4',
|
||||
318000: 'f42045295ce30bd4ba9fd9c845bc1228949b4152517185d3dd0598d8bae38481',
|
||||
319000: 'f488695e46b53e4734f4eb9fced93f067f037af7099d0f9510856a898a0c438d',
|
||||
320000: '83e42370911c70bfa2ad89a82ecd02bbc9cb13d24c2a900a0030771898ed618b',
|
||||
321000: '74eabfbb64eb89488b59393ed529b4ee97e0f88a7fc14f798f22715bcee3fd19',
|
||||
322000: '54f42438cf96fe8efa0180a47350002b34ccfe82123a85a9a6dc6f8b0894f0a2',
|
||||
323000: '5509357b42476732435a1d5a6a850e27f62cb3ca81219a94f08dded06dfcbbf8',
|
||||
324000: '1cd477361dc4c72b4f157469b6f3616e060f5d058b44ba864225210432b35540',
|
||||
325000: '060ae9c541eee90a80a11fe300583e47c0ad4d12093d0651310cc6c87ecf2b75',
|
||||
326000: '13ce59868c947ecf2d6aa986501c9f8411a59b4e0b8b6113f4f45dff4074fe59',
|
||||
327000: '16a1532ff983f5cdab6c2d465d58f0369665a926c9d877212d95f9387e8732ca',
|
||||
328000: 'c9608a16fc43e7e94f04b3309c067b0a4f68a66705009dabb39898f41fdc862e',
|
||||
329000: '029b02acbe0fc8610894f0b89d6bd274362195a443fb9e5e4d4485be7786d461',
|
||||
330000: '01181b9529da87eb39510619285c08b05cfa2ca2c7a2fb3ecadba474c1db3aa5',
|
||||
331000: '2371752c35da119184e53dcb593a193fb1c81d6edcff921599328a6142a57e3d',
|
||||
332000: 'fc025a8f319725faa9903eca86935116a2aad8fec6193055d5d5f0f431ba84b2',
|
||||
333000: 'b602651afd9988075887a3f48455d977357032185b7d482262efff9cc97a45a6',
|
||||
334000: 'a9a2bfb6b0f8db298b5ba732f4a23242fa89e1c5038916ccfba3751f7dd1b4a0',
|
||||
335000: 'eeb3e35b6ea94119bf058e6b02a2ec2a706089bc1c360ca28d1b4e2ce1b547ac',
|
||||
336000: 'e834f7ac6bd8e5b46ecebda4997ae73bccf353fa024fbaba7ad4d1d70d08d14b',
|
||||
337000: '229a7017b6d0ffa08374e297b07f12620bcdbaaf5977dd39f3f207ecb03c7adc',
|
||||
338000: '83d86500e8b2ef46a0f1505e29f7401b123ea5ed5feacb49864ab1de284eb574',
|
||||
339000: 'b08e9dcef3e38d631414b4d3a8d8b3dfa8a07d9e5829c515e7db2fec64c60827',
|
||||
340000: 'aa36ce6494df6905f4516530ddca77d88341c0202bf052347a86e090c03ab0ad',
|
||||
341000: '527b50125d5479117ca8bfe84fd5507b2312360908d4cafe8588d40c4ef5622d',
|
||||
342000: 'd1397e8b0f847b6d46c409659e9e10f3182ad9bdbca563cc2598af37cdc080bf',
|
||||
343000: '896576aef0e0dbdb2df8291f2832716029f69718ac79f355425d475a1b2b7b4a',
|
||||
344000: 'cf4b3c75279b0f7d71cbba934228709ff03e3254fdd83a47fde66e83ff393eb3',
|
||||
345000: 'e68c760edf370e758e815d9363ada456fa2568b06abf8626f28a32cc62944770',
|
||||
346000: 'ab19fe52e780e45a716ed97e93084bb3f7916e0a4479f131c7f5f7a2e7d69d9c',
|
||||
347000: 'bc77d338a10fd71daa6c68fa9df2fdccf44b6d9b1d7c6456bb281bf8547b622b',
|
||||
348000: 'f84469ebb26540cb3de0ae03f1f47452f9a9bc6b7195c3b5260d48623507ed86',
|
||||
349000: '5bafc3e6dbe4bf88257c4381cb4f2fea0a4cde1f272e1f1006600b507cb0f670',
|
||||
350000: 'e53ff96761b9c98bed635457acd1856a2ffe65b84e339cea94ad30f13a31e5e9',
|
||||
351000: '5e0eb50d32e90e5ca73639313e5d3a850a113fbe60ac494ca6cdb799e9604d7e',
|
||||
352000: '1a1a04aaef110493bf8fbdb8d1784a7706c6e303773ad0bad6511dd3c1da6c54',
|
||||
353000: 'd72f3b615e244584f131f2f8e7154534aa9d2d9c4a710bb4de44de556b5d1232',
|
||||
354000: 'e81b823c64229c555a50e90a4c3c4654b6673cbff4c62c6e897d09aac00d95f8',
|
||||
355000: '11eed14cac2024588b389ddcd54315923ffff03d6c4c5f32657ba551fd308516',
|
||||
356000: '465f28c63e564296f75a4a22c3ad16714cda29d7a2797e51e512eb1ed5f6a49e',
|
||||
357000: '2b544c79c6f74ae1c05c268669e57ad41d6807f1d0ce1a5024b7b08646df6861',
|
||||
358000: '8a76eccf113819aad8d7c17a459a614274bfdaeacfb4659d945f13dbf3907ccc',
|
||||
359000: 'e63743f458f1479593d73952bfb882c72e5889331add576a716761eff2cf13dd',
|
||||
360000: '0ab0bef439debf08dc3e1df9010c4e119d6862de6620de5f725cac0013c06301',
|
||||
361000: '50f649f0c8bc0d77ffddd53fd5b896705a8858d3684a3ec876cb320ad26d2c29',
|
||||
362000: 'e058c43c366c9c307ca3ce5a732f0ab2b4e09ff291a067366af25a82fb3cddf1',
|
||||
363000: 'f99d7f0ec93ec7fd74273bb6f8bc4ad8ec0fcda9f6406001668dcf62a088a867',
|
||||
364000: 'e5a69ea8763650542b1f90d6217af00da137a9bb72cd0dbd5379570d1fa696f0',
|
||||
365000: '5d0eb3063474c6bbf9d9581898fcf8c4cd097fe5c6e3875d7cd701a5a0302507',
|
||||
366000: '14da2978f4d4eeff5eb0a6bb663e514095acfe2210039df8dda9b76f522f74ba',
|
||||
367000: '4ad3ebe23fe9b0f407c0765adb69ee973de5b84db29c97167c7a358fe8696bfa',
|
||||
368000: '22db57ccf8c8193ae9fc08abe11f1ea8e869bfd6ab2c0b918820acdaafc19136',
|
||||
369000: '530052d61878d20282a69e0bace41857f7edea6d3e9e92fd0a7781c8b60d208e',
|
||||
370000: 'cd057d0c889c78e06d0eb71287c46deea4340b8c7a4c470dd69020e1a9029c43',
|
||||
371000: '41f32ab47863f0c44b8755ecf9b201aaa92999051ccbf8a5acdee0e12dee4110',
|
||||
372000: '37de29bd97d75a41dc3eb52bed9c09baf069c23120a393888e15e61ac7f81ea2',
|
||||
373000: 'd6b26bb791fdf6d79f59d5c37ef2af0a35149737029157724224d614477527f4',
|
||||
374000: 'de18c6a921838385950b10ed9429579e95a0bdc574f448f457ee486a59a04260',
|
||||
375000: '1f86edf4ae47f409f8de6a16c17c2da063937478a0ccc800633e4d33b531709f',
|
||||
376000: 'a0511050ef086052fe5671d3333c255507b03edbd365cd7ce7cecddb4000f391',
|
||||
377000: '7b8f0060f51fd170296b9e000bb2970097f77ebf0b5872a9ac01be5aa03d7ffa',
|
||||
378000: 'fc5669d194d822c55e141ddc9aa229059e4663169456dc927d43b44a2c8d3c0d',
|
||||
379000: '1c0bf45f052e0123d1a6806fc4a93ac8261ed26772f324c1b3e7c3fa44d7f442',
|
||||
380000: '73b69dcfa0a916a4c495a14d4d43e270b123ba37d267e3eb0d65890336fd3e32',
|
||||
381000: 'e89e3d548193df667c637f774c76f508ec79b7bf1f2ac5d9adf7468006e34454',
|
||||
382000: '6b835973c2efcd564fbd58d96226cb984d5c32dbf63d5b6044a0a11f77c2ccc7',
|
||||
383000: '50b7011656c5903941303c7a19e4ea0fc6bc10456ba85638b0919658556e47cb',
|
||||
384000: '74c509b238c1418ae527d370e8fe922f7639c88ef8d78fc3a45894deeb2252a2',
|
||||
385000: '6f9f6147b06d8fea8c440cdebb3869b7bface7cb2355d66639060056adc9bcc2',
|
||||
386000: '5b59e68fd62751b8be184a5621f1c63f1e9cbd22051487999816df392dca7837',
|
||||
387000: '86d8b9628b83ac63637c86c364001067ee1c3050f395758fd3f0bcab20cef703',
|
||||
388000: '5e570f7b69738285848bd8b7f2fbb82d05e99a72cd4a0faf20a9e3491d2d041c',
|
||||
389000: '78d31bda70842388c1cfb2b83e9e95ded37a0304aaed3292e2687a96c07be9b4',
|
||||
390000: '316da5e2ce79bbc4f4500ea6417271f0ae268f85f86f81223d0af38d1e0e0a67',
|
||||
391000: '45cebf8a25c25aaaac06e915fb3fcc99a84be61a266c2bd8997bd1fe00a7321c',
|
||||
392000: 'c5ff6e24f524bfb1f629245345052c0a0df33ab881ee9d11dcaf64915b2acad3',
|
||||
393000: 'a54b5d4824bae06f91566b73c99a07b812095d7c133d1042ded80211c0402b3f',
|
||||
394000: '0ae76082f8347ddf838ce0ad486f5decf6223e8b8cfc16dfe178b15830807c60',
|
||||
395000: 'dc436c97ee6ed427687e6d48e609d271b125495afdebbcf611c0fd5102ec900e',
|
||||
396000: 'a87871c13ca3119f309a4ed98e4224d4eff01387ea67e86148007ac0d759f2b9',
|
||||
397000: '58c143b302132ad4b38718ebe1c5fa81c1177a30045f1e4a52dfa749e84dde1d',
|
||||
398000: '33089400401a190c9375abb63c6ef489123f7369e5a35766b59e69edaf4e3b93',
|
||||
399000: '656305fae087b86758bc7a01e13051b1aaf4a5c53d9b71e56614217a82b20eea',
|
||||
400000: '7ea274642eac11d02aef589428b181e1c8aed913e248b357917604b00b23c752',
|
||||
401000: '1969a9f64e316e0b4fc864fd7162379821510a944652b8dea747e425c6222807',
|
||||
402000: '3700e1703e7803a4577de7e364a34c3be94d3f933e05c5dfbf6ff68349b6fa31',
|
||||
403000: '8b622599bf4b2d82bc11c897f35536036e6798ac101a6256e14477624147efe2',
|
||||
404000: '5f6f6dcea393ddccdec69b30881ab08e01e1a7e6a50d681883c759b742c12d6e',
|
||||
405000: 'c3fb0507a57da69042e246dc8ecac101f33cf2b1fb53fe2f507ce28a0d82e4b5',
|
||||
406000: '4f034663bb7b84439fa256f7b4ee2a5cf93e0048c1eb476aa2152453ee7be544',
|
||||
407000: 'ff99ff38df6f928242571fed5b03fe0c3cb481b510d3c943e47e26f26440cbda',
|
||||
408000: '89865af5e99e64cef61dd931061bbca7fe355218668f487f06474e46ebf892d8',
|
||||
409000: '1b1d050c0b98d55d9ecb0d1a5b03e75caa352b4e0ce6a57136551000e150c480',
|
||||
410000: 'fe52790595db35c7685e5261de78d139c830af0a5ae73a7e33da21be9ffcc9e6',
|
||||
411000: '1afe55e66e6e16abff282c9d6bcfbc6a3a6e40db3c60323cd44e3b3a5d4ba924',
|
||||
412000: 'b59a4057d73e8037f3e0e9b92310def7900c19917892ebe7689d5f8d77912388',
|
||||
413000: '603f3ca16a6a795d5aa807eac5d6ee7f80a1e2596cd91cf41cf90ee309b2871a',
|
||||
414000: '10e255807a43bf5b95f9af648f1724de1e953a5540819822be5de184ead24ae0',
|
||||
415000: '9faf6b1c33006e3da3ae2c7599c7fdeb5078a8427edd6666290e32aea67f5a23',
|
||||
416000: '973b77e4d491ce0c163ca64f5419b05822eebc2f5d85810f51370f56e0d812b6',
|
||||
417000: 'bf99710d77fe84ae3ef5a371887cacc2ca1fbc5549b04ef9f61cb9d911b06a9f',
|
||||
418000: 'f032acfc2d629fecfa180067b4531f9d72924c5f204cfe9053c6556a334373e8',
|
||||
419000: 'bc9f085612af6a21411fcf51259387d97e6ae1169e4c6f92766a168302a98bac',
|
||||
420000: '9136bd1fa7e8850c52e3cda39a96f754cdc605a69ea49b7d6d959d0cce81efba',
|
||||
421000: 'ec2a78bad1b4b7325d574122ad8c3072d27fe6854a6965e6917a28e0c868a229',
|
||||
422000: '536f4cc5d6ab082d32ccdba1caccb5d4f3145b60ee8538b1e18c6231dcfa23df',
|
||||
423000: 'ac7ad1f97da0e55661bb25d53c27ea72f705d54050f020aad9fd5856a27117cd',
|
||||
424000: 'f840eed1c0fdfba43965bfc175e4d92a5ddd0fd6022dfef988d70a5e7f363116',
|
||||
425000: 'c1a8f9b79a34b76c9ae58e005501dbb1893cfbda8bb25b0d54854657b3ff1a54',
|
||||
426000: 'e8895d8f797026a2f19ade2b548bd2cb655e2d756dcc0a50850fd37148d24ae5',
|
||||
427000: '0ca56cf4fd882583854fca00e339534346d53178c6679f8eb1e7a5dbb7b9e7b0',
|
||||
428000: 'd3ce901843f952df4b63e755a0f58924ae11324aad47afece9a93502dcae7c13',
|
||||
429000: '774cf2d1ba62f20f5e1ce50692f65f0ce35c7456d3988cfbb2e1b753e0ab0f51',
|
||||
430000: 'a07fc310f6201c30f1fa70a082348942c65743d51e3c19492596f6aab4bc9bb5',
|
||||
431000: '7bc7f405bdfc91dd01877ccfd6142c973d81dab12fa188f468abe6c9a30fe0d8',
|
||||
432000: '7fe40e145078c1952f87547156e2147144449971f108164d27bcfb93f1c840d7',
|
||||
433000: 'dbe52d2772d6c0461e53357b71a79096140039ef643876ec2d27e1bdfee7e81f',
|
||||
434000: '94b379e842d4979bbec171bb08881227ac4e9477186ab63f7eee9b4261b92365',
|
||||
435000: 'bc9cd265de563cfe80090ce12085539bc67d406c23e4d41751f191d6b6c63168',
|
||||
436000: '3ff03098d2c20b5cc080363a84d8295510db937272801454d429c4d17a1dbe10',
|
||||
437000: '5b8ebd40170e217eb66990af3574b80f4e66bedd092e44836aa5101ded50c13d',
|
||||
438000: 'f725813181d50c0b20552635ecc578685ab0218619d53ed23493c46fb060ecec',
|
||||
439000: 'cc7cb296ae79850ba2f6f7532c445189452831957f62cb823bc83aa6b51daee9',
|
||||
440000: '757b816392227c9d1a14f7350cab6e77a46e465e7853b1291abdaebcee2bba94',
|
||||
441000: '834515861eb72aa472843a6924fa9d6350630ed56e35181d9fe66e3a46b6952a',
|
||||
442000: 'cac31ac937ebb6f0cf4f3c33f19d66a05bdc11105a98ef12c32b494e19e772a0',
|
||||
443000: '552b03680b5495c7bd6ec58db0b6767e6bde5c5961a02b7b2130ea6116bddbad',
|
||||
444000: '2bea6ef419d9fe81de8bf5cfe21b2603f202e2b9a959d2cfbdcbccbb7a7406c8',
|
||||
445000: 'a4106f2115b356eca92e5e3f45b8ae0e13b82af94bae391942200d1c59d34d17',
|
||||
446000: 'ba49924afd8a40260e546e5e4633e945877ac9d5754c277b77bb154c0c9d0e40',
|
||||
447000: '83ea56135c2b052df0fb09c68cb7a085d876854f34338abaf3cf51760c83c3eb',
|
||||
448000: '806d77789673fa09d24c188ee024e71425df7baf623fda073076f3a9b87ddbce',
|
||||
449000: 'c41d60b93fca1d2adbbfcfa99475a8eda152630d815eb1f6c58cbd15a0c2eb65',
|
||||
450000: '732d315feeb485fcc5055916df17827d80e7f831c43df81ea79123882c4fb356',
|
||||
451000: '202156d29e57c51941cdb7256918128589f5b8dbae5762bbbeacff6eefbbfc71',
|
||||
452000: '0194e8482252f2492e1e2696fa5761f50875d0188310a0cf1d967edffb1b79f5',
|
||||
453000: '9e63570821d8b5cd7de0c5f9fda67300f531d87cd8f778fdc6bee9519b77e01e',
|
||||
454000: '632353fe048e83ebef942d64d550721d29b3fc8083a247b2443f2aa2b706c6fb',
|
||||
455000: '65863feebdbd60d16330000cbae496667a67dd5fa8b8e3b6dd3732843f6222a5',
|
||||
456000: 'bbc554cb3c0281d6853ff2f6c25b1251eb108c09e5a6c4a69a56e8f79c9206e4',
|
||||
457000: 'fe6fdabb84ad1ffa807fd55db29dc95dafeb0e9a5eeb5ce75be8a4d16f9f116c',
|
||||
458000: '949bd40d861e4b297352722b6f01efe9574711fe64b10c2e8dd769ad99cf9655',
|
||||
459000: '9c139c9ccb41fbb3ecee878cf47bb37e28c76816c257f99dd7b23ec1f3eb3ab8',
|
||||
460000: '61c3126d504fb38ea21948d2837368441389155e9675c7f1629eefcd03198f8a',
|
||||
461000: '28ff145fe2abc7cacfb1613d40b45e13580c643ad68c44db81d15125324f4f33',
|
||||
462000: 'e6b532924d07f1ff3976a53397327ccb232d3b2ac6ab090874cfa6ae47b8b315',
|
||||
463000: '15ca651db9b159978ba6b2adfae1669e3042f4311ed683c72838bf337d71274d',
|
||||
464000: '6903539d4f38f728e075441efa96b8013295ac166db50ddd4103fb2bc9937f0f',
|
||||
465000: '6b6c2640411d900c13f5b53a9a993d10fbbd7f03f6fd477e120ba06a7cd67778',
|
||||
466000: 'a05d282fcb9c091a0a96c2ddb3728c1e98eb767fd5311bcc08c9c10b5a0eb24f',
|
||||
467000: 'afb81fdfd562f2cd2d53434e6458c38462f85b65958ecade001bcf2b49f9b28a',
|
||||
468000: '97374586191b3a30654483722f7c6486ef9d9f71aaca44b72146c11d4f4b80c7',
|
||||
469000: '90a829ff58be9ad30024ba1851515cd96d567be1d99154cb966067316d1902af',
|
||||
470000: 'e9fbf2d2291e1b5af5dded3c223611d9f2e31fa6e1d8c075741aa155538bbddc',
|
||||
471000: '3b4513a6a6c05ae9d2ab880a2d1aed3b189e0fca9cfa2bbc7c31ebcbffaed9a4',
|
||||
472000: '32a57820226f672e695823620fccd42a6fe624dd2d88f7e7c76e0e9128883b27',
|
||||
473000: '15cb7e920799939eb01d06b1b713cba3ed31ef92ca037ee98eb8101b6331e5a6',
|
||||
474000: 'cc23e76e68dd726f31e592a4ea5d6aa3cacd827d068317a0a476f16ce468d9f9',
|
||||
475000: '9e7e33a8ee53a4bd94a26b35bb43054b42a238a85a859ed9e4a33db978a128ff',
|
||||
476000: '23c85e4d8edc7939fe0e5a47fa250f0921b366711c34910c394af00a4efd90c6',
|
||||
477000: 'fd9d489fba4175c1fb3b8a0f9bb947287e64da4bee9f8a7fe73411632c820368',
|
||||
478000: 'e287b69696013e70b276843e83af6cb5cd076000cb280e43de29cad7f706dec9',
|
||||
479000: 'e86c7a06d96887e404c052b0fcedf10d40423b02d6b9fa7ccb9827551e5d089f',
|
||||
480000: '74e7ad7d78dc585a5cd1054c0288aefcec0e031cd40f0ebaa0fefc7a325f212a',
|
||||
481000: 'a01f98758cd3e561d02bef30c054ef70405ebdce9e670a7d6165efe3a9030886',
|
||||
482000: 'cc9e22f04e4e2e4f52b8382281092ba50ec99722c9172a0e1c9ad654d26bd4d9',
|
||||
483000: 'c92e278d54111ab2f3906eafab87fdb932307e60836df209856a97cb9e1ae3a8',
|
||||
484000: '18fc89d07631aa01891aa44231dae6cfa43fd4733c193468b033532a4574eb80',
|
||||
485000: 'ab546bb694363e07469dec48b559479d34d724a43865fa8f3a178ddc7b83e8a2',
|
||||
486000: '59926fddc89abcfb55775a58e5e6e453e388f530f720dc61001f1ac59b59cd43',
|
||||
487000: 'b41bcbd91984a3dfe72c87bb7c2ef32ba3eb3cf4318f5d37654459143dceca1b',
|
||||
488000: '94e859b8f01ccbe9fc5bea5c274f252e23599cd444d5eb0962ece97110b86eda',
|
||||
489000: 'fd99b951714162e55d8944e8c3ce9b098c4b74ea24017e4b204f95359202df08',
|
||||
490000: '80b84d18f4bc6d624d4190db94920976926bbe48f2e3e0f4c08106b9f99ac5df',
|
||||
491000: 'b0c5b14b01225b1ed8272dbe9b2def41756cda0a3fe6a1a8e75b0795b72e6a7b',
|
||||
492000: '572dfe57507384278def462488416248a3039c1a76a8c4aa23d1ec706de1e3f6',
|
||||
493000: '75f22a8d4d202914c0f3bffaf95e47eeb82eb00dcd2913af5dd67acf6f8ed8f0',
|
||||
494000: '626e26524f9427582d61f340ee9181d8f238ad8acf96c1862724fab62ede8c8b',
|
||||
495000: '69a11346ba65c79b5fd566768a8b2880a5a29e2552ee8e343bc88bcf43e9ca54',
|
||||
496000: '314c88db22753d317c7b65f872d54d3ff672d04b3781293d59a1ea902811336e',
|
||||
497000: '7bfcf2614b54ac3808894694e8b15940d6740760a2252ba4b47b2926bdd4bf51',
|
||||
498000: 'cc02f90d2fd4e0f378d2ae3bf1011026ca3d7f6642dab316f5fbc2f57aa840d0',
|
||||
499000: '841182d8519cd7b2d8fd01ddd087de892ecfafaacf9d6aa029ce1ed71ee0d538',
|
||||
500000: 'e2742854018ecbfef71d626ab797b7c6137e84ff1e27a05b5da84b2fc7c43145',
|
||||
501000: '07334852d0b542f897164e4d74e699678c850af4e46111d495701a312be21105',
|
||||
502000: '6150771ac25f4185eb94ef6aa15e7f409b6beb411358ba7d982755655d431492',
|
||||
503000: '156d3165eb11d021c46ae5968aeac20f65eb05e68d841bdc98dde156d42fb500',
|
||||
504000: 'e9d474e59bf527d84f031ba41cf471f49e5bf0472ae1b3f48f036813a0b195e6',
|
||||
505000: '261cda40eed5ec5b2a44edcd995b24887a45a417fec3f68507a4d600ef2abc33',
|
||||
506000: 'ffbeca24a8f2b7cd6d64207911e153a2f0bb70d34bb4f073c33197e3a2276558',
|
||||
507000: '14e2664bed708d57478f6fb5a4228e4c26843bd6533056fb38dbf1e7595028a9',
|
||||
508000: '2750e25c3dde233447266f7b35f78d8bff18bbde4ae094d06e38fbf6cd2c6902',
|
||||
509000: '5fc210a72d142dd413bc3b76d5ac772564a5917aef13890e7cee190c61b059d0',
|
||||
510000: '7184fa5628bfe59481bb3542bb4212b535c3ede3c24425f0bf7b1b453c238c6a',
|
||||
511000: '2b4bec04d73f0e524cf12d04bbf7b3859b489275edd047f41b139ee6220074de',
|
||||
512000: 'ebf81767f4d26c98bce62a5f5d1a289da5bba770e88f346e2fbb1d676c155e09',
|
||||
513000: '8bd8a99c65a258ebec809ca76e2cc91b0b4fec5dfe0970736d2fcd706aa660ff',
|
||||
514000: '2dd7cba17366c483819ea090d3196950ee35a4bd9f28acd94575525c851234d5',
|
||||
515000: '33ccf5f5230b9a79cd89eace91cd1c2858f6d040a11494ffb8d51ee934753d57',
|
||||
516000: '30eafb000d199c8d9d1a55c5b09a382a3b10bcbca837e584db6647699ce6a3c0',
|
||||
517000: 'e7ed622512e2483fb8537579ce2d2519053f9a6c8d19c42d665007632d5f74d3',
|
||||
518000: '5180d11f98a25b8698b969c3c9a4b736522397859e7dc00665658f93c6cb8be4',
|
||||
519000: '17ca81b3a3c942c80384c4d2c176b5274c52569b90eef686e077f53a884130c7',
|
||||
520000: 'fe1f552778cd5250ba6556712217b86fd6f1d574143f8ad6238e1d6b67481f97',
|
||||
521000: 'd7ac10bcb8662ab9940c4212dbf0c4b9cf9750e3c04a905efdffbce63f065de7',
|
||||
522000: 'fa7625300aa890ccda58268eaf2daef14539c195f93311099754f955f367413f',
|
||||
523000: 'a55ef67a0e75bea12003609359443a28b42b9037290872d6b1672433cf5c0d34',
|
||||
524000: '08d1d556393bbddf6eda11df852a792515ab9c1d519bd91c48711a0de8faffcc',
|
||||
525000: '76b8691b16f7dafcf980ec3e7c69c2f3caee32057ac8d61b3d3aef0fd394642c',
|
||||
526000: '6d3ecbd7132a725e330f0a566d8d602a9cd96326a36387c1c1fe52f7f6ad975a',
|
||||
527000: '71d08ce7aec6ff50fa1ddc37f6b1574db6e71004e8e111f04a9b715103f68e01',
|
||||
528000: 'e9b103c11db50a121c9e3a1fb4cbbd4fb124b07a67167c5a461a539dac3f932c',
|
||||
529000: 'd2f784907a019c1eb5830a74799562cd6d21570c4591bb038f379897f3296d75',
|
||||
530000: '996bca5a7dfc4a8c7f9f8dfb38b1881fb97893a950f4c8f57462506782ee8ca7',
|
||||
531000: 'c84dea2e7fb0c9c884f8dce525b8b65bb7deb33ab7fdbcf72a5382e9bac2ade5',
|
||||
532000: '3e200d76a92dd1f6fcc420d44ebe446829fa5e0162177acd75a13c98f1477cd1',
|
||||
533000: '37fa7cd81c658ada13d04dcf0e674017c6b7430bb0cd5a080b1b924396377b52',
|
||||
534000: '9483c344293c376d2bd247f2846a5ff5038b27af48ea5badcde101fcd0113dfd',
|
||||
535000: 'c9d4f9982955c07960da1e4f92803ba135f4584416e0767a3567e4d2a21e62dc',
|
||||
536000: 'bc9abe4e4be06aa8f61b599d64923ba88d36886b638f413b12bf87ae852e4010',
|
||||
537000: 'df8307b0097b4882740c18432ab2d7e64486cb7bb70b503923bd630ea277e9e3',
|
||||
538000: '6e2accc5e65a85e0aad098dbc7021bba3cac37c70f6a0ff3135f86b90954bd2c',
|
||||
539000: 'b7d9f50727cb512dbaa1ba073d227805e7adc70f9aee46b9721a65ca3d34a8fb',
|
||||
540000: '19c695a6b8bd256fddfd00c53e6401829ae59ebdc1e71ac234f82ffb2991ed0f',
|
||||
541000: 'cc07c3740f96ec8929c7c8714892a19815fe64f4bcdc4d1bf12cd0a9e1e82125',
|
||||
542000: '23ffec3127ec3e093ec8221fa760510e63fcab31f9f0fb2b48727649e4308980',
|
||||
543000: '6c73b242bb05af125ce4ea324d921ece54c9510980128605aec23cc465dbeb00',
|
||||
544000: '51f215e5f11ae31853c4caaae1d01aac2dbafd00302b70e7198e213d485d5bfe',
|
||||
545000: '2c36150f62c86d050e4c64d83aa0071ef4af0bf8e1e17623c740ea4ee7f3275e',
|
||||
546000: '322e6d77f529412c4b7a1c7778253900f1976f238f96b771845b56aa080f4d23',
|
||||
547000: '1d3413530079389231bdba0a0a7de7c85a4ca991569b2f26a1e48ad2f34a995f',
|
||||
548000: '3576ec0d3ed469c5498229af531fdda290777fa1ab7cd83c49450f730c5ac0d6',
|
||||
549000: 'd2a5ed3d920f273bf891ac40ece534a56f8774c8809ed7d8482ce4b78bdf016e',
|
||||
550000: 'e1f75701ff829339e03c019eef2da1ee4239f38879173a833f81295200e96d29',
|
||||
551000: 'fe8339fb53a33e3b5c4a39cdd64d869f41b4fc88c73fa018eb5245d2a7a01e94',
|
||||
552000: '8c2180cc834d5c14c9801772caa65e82b44cde33da7aac586c81b2bfe77412dc',
|
||||
553000: 'db9b4ea2cbdb21fd48a69da3bac2dbe8cea2a16d45f9252776633661e28fd1d2',
|
||||
554000: 'd09c32f70dd94d737205922618b789204a39668b59fd3ef405ba6d4ba3bd8aea',
|
||||
555000: '6e8efd0190d866972b469cc827330f84f515d6466d8748fc33520e8e0505bf3e',
|
||||
556000: '8ce7cc8aa6ad30971014b332c64ce87303098e64391a9be03d1c621df851c00c',
|
||||
557000: '9959211d82372edae69cfac47cc5c937fecbb6de9e235a5a3954c653f3b1fda6',
|
||||
558000: '4f1d620babaa28a03d02c2ba8b7d1526ae47d45bac1220a1a794cf4a489536be',
|
||||
559000: '3a0f1617c6056a27b1da33f0e4ca80b5b6648341551c19622f7f4b4f016ce8ce',
|
||||
560000: '54877adf522c9abaec851a358ec6fba9957227e0e306d3a425167685380cf6ff',
|
||||
561000: '55db8f8ad8869221ee891244767a7698d48ec61264729cae19008779dadba422',
|
||||
562000: '4ac9c41b29253ed03c45222412c53ddde7b9da63441d06ea91b448d89dc220bb',
|
||||
563000: 'cce11479fe1fcda6af734a03be8ea044df7e2726df67927f5bc76c33728133a2',
|
||||
564000: 'e2e5c52e028ab1be7f5d182a217e16694ffd370930cdafa9443b627e8da256c5',
|
||||
565000: '81488a98ee93b0cf5704cb1f46383456bf5238fdae7bcde4d0826276b5a48235',
|
||||
566000: 'c9afdc804f7b0df600c51b9d596d6107a0cb71a4f8ed3ea764585ffc5f70644d',
|
||||
567000: 'efd3ac4baf1c6e1fb2f8bac40f1416f189c9eb1135330ae4c25f296f012907fa',
|
||||
568000: '225c3db3b83b78fbf9459c7e1e53ae35b01752608d329eefa99247d8a2b9b83e',
|
||||
569000: 'ac98f3fb9084261c19a63fc8f10baeb5b03aaf9ca432d7f763a3a6706f81eefc',
|
||||
570000: '4ee2eae15c9019466bb50d103947bdf60a5397f8fff56154ef782fd1b8579dd9',
|
||||
571000: 'a89f29bfc76988560ccd59a61f8a323fde7c4020abafd577f88d1b0b4c289260',
|
||||
572000: 'eb8380e096c79806443535f2f3b007378e43d2cf3ff56f3b7e6a1b2639ff3da2',
|
||||
573000: '3182ec1555aae0df8d76dd5f37bebac6936e0e0c5609263263eef2e6e87863ef',
|
||||
574000: 'fc45bbadf22417d49e7df89f02df358b85a4cc07adb4010d3bc0334f397b01d6',
|
||||
575000: '6faadf2b0c3a98952bb047a58b83466a223dd323a6e32d68be5e8d064482fa84',
|
||||
576000: '68465ee1d64b65ca71bbeeb611a8b4657d3ce765857d049536047e2ea0490044',
|
||||
577000: 'a019fabcf35e7d4fe15e4e3a499dea2bedc61e4c9cea2803731995def15a9794',
|
||||
578000: 'cc2fb6111fa1a338eeade31574b77a1012ec5948f88e7e5276f7ba4443df3db0',
|
||||
579000: 'c5e7c605c03e56f221397f55bcd164dff21856f7fa48e745e61f2d05d7de5f54',
|
||||
580000: '110fe7150988c930f9c324e00e108050b8be5ca2eaf200f54477073c72feef2a',
|
||||
581000: '0c3adb09dca9a9924fcfe30f16bc70734ec62fd4a306ce65a38b74c5d817ee40',
|
||||
582000: 'fc47433896c47765651d8cb31c88cbf80d0887bf071eaab9d2963af9f8bdfb56',
|
||||
583000: 'c7c518f8634bc7c5cc536c9b77fe119cd6cdce4fafb25d4c0aba53c5bd970e35',
|
||||
584000: '43b2357dd053608b30488aca98f5d9c151a68730fb8d008f1bd1488bfab0c391',
|
||||
585000: 'cdfdb4ebd1c5fc3689b4f013254c832da7bfb12f3440fd327c6b065afd093dce',
|
||||
586000: '48979b814cb6964b20ad8b322c4936c7b241c24fe6d610181c43d1cd003e5d90',
|
||||
587000: 'c262a418368aab465facb8f1a82a392eb74e3fd56760779033a97a3f930630ff',
|
||||
588000: 'b20dc4e0c17a223408a51b958b6b481ddeadad513e046c86c7418a8e96e35992',
|
||||
589000: '0411c5ce49578218dc8e225c11276ed9a9802aac47d8f5837c4eadb3bd72e1dd',
|
||||
590000: 'be8fe8b058722f4919af2834c076e2bfb602654a0343b57eb568d0ed4833fb39',
|
||||
591000: 'affcbbd2a9fa43a64332412d649c2c0bd4b85e90e5e67576f09ee0c207a66cdc',
|
||||
592000: '14a2d414220d240286ec6eddab0b0540ff4d1794b39f7913c00ab005e2ca2a3d',
|
||||
593000: 'a1e2744404626b5e8e101402f50eb751c282ce692d58493ac6fa2efa43254526',
|
||||
594000: '679da754e8f4cfd017ed5c8111361c0f9624104a87a50d077dd7bb5bb97c7ddb',
|
||||
595000: '0c31eeb25d718f96c68de13704a0904c3a524b5936690ddab8b4b38fbabad5f5',
|
||||
596000: 'f930b6516bb01d3365bd0e28aed56a580b4efbb45b0abed5ba941f7a56b545de',
|
||||
597000: '56287fa7bcb53be966ba75c15b92ddf94a7bb51f4ef51037e56ccca936df4d41',
|
||||
598000: '1a6303e68038921fe23644dd10f5cbad4af3648f7cbe8432860f439c84fb226b',
|
||||
599000: 'add270bb4fcb8dc9c254c9130fa037dcedb8ca922c2e35ed403661a896addf5e',
|
||||
600000: '786ea387bc7e1665655f1f683d56c67baeb9bc482bbe4401acd167793d34a508',
|
||||
601000: 'a4c5f398f28cfcdc6e0b6fd467582ff5abfcb4f003cc59ae7703e9467cdd9ee4',
|
||||
602000: '21b19a5a23344325581e7daf0a398b37cac5b4c335237bb8e6c14e270bdc1567',
|
||||
603000: '000eef5b4300cfd804554e199ca6ffcfe73ec8446e82ad32abbc51f45594a368',
|
||||
604000: 'fbf77437dfba9caa4d11b03a3673f2274dbd3d92e704bc5e8ed3f90d99d2c988',
|
||||
605000: '0bdba251f44baa12e2b33b1ba89eb8a5d31ecaf6193b836e6356845149ea3f33',
|
||||
606000: '2dcadcae1cdc68ab6404606937aa9981400e12ae77474462764b8077d391d2d4',
|
||||
607000: 'eff9dfe7fb3e31536dc12c78603f023f8f1f59b2b44a59359e8bfe50b2e22299',
|
||||
608000: '50fdd1223efaaa261f6fe592d7b9e1e32b5a50f78dfee7bc8af19250f8a82776',
|
||||
609000: '5d88ff14c7c36602251061379179ed87c5f7d60d746d8d10fe678a0303ce6596',
|
||||
610000: '5e56249989c623bd7f818ac1e64c5d1b1f3afec3678559d446e356ee5bb3394a',
|
||||
611000: '5a9b15a38dd35b9ef0a028b12ab69b0993eb4bc5577d50c486fdfd032f243a14',
|
||||
612000: '258213d8b7141418f22e303ec43005bae0508a3597fa99c4d59d656d7d768af2',
|
||||
613000: '5ee8b46ebdec5576550540f7fe01c5a91c3ba7bfab26db605ade4e620430cc21',
|
||||
614000: 'c41a592e2042d201bf028bbfef484d264a391db9dc9ef302708e7a39f723cf5c',
|
||||
615000: '70fa6591c0b7bb63665f040b87750668be47058942c79c863bc00232f536d725',
|
||||
616000: 'f4932c1c8fc84e3f4b97336424e27eeb09a65d7347097e983e1ab698a38dcc0a',
|
||||
617000: 'd82bd9b58adf54a50bfd471c66144f2969ab443648fb7bab8299408afc021544',
|
||||
618000: '72f7dfda9bd67b900106e632cf50204fe27cecdeea51d9b0cfde9ba6da8dcd5c',
|
||||
619000: 'ac434b8d723ba498bee878cf589c7d61b74c7a00ea5b7abe70149ba878522332',
|
||||
620000: '519930116d8a8f3b01ba443bbe4fe5346cebc783c632cdd5f62d305f61b6f446',
|
||||
621000: 'fed977a0174f8cdc3e979e7435ffde2f7dbac2d4e1c63593117cfe81f320bfec',
|
||||
622000: '1a6daef00425e6ad136d7bda36cad99baa762f6d788b92f630ed1af33995211e',
|
||||
623000: 'b9f1e2836bd2acae3a9d21516052007005f3f67ae1fc133c1c4c336abe0dce51',
|
||||
624000: '5d4a19f7e7f99b5c39bc844b0a90ebcbd97d88f817f8f2edafca78ae499c3167',
|
||||
625000: 'c1afa80aebb856746da25503eb786f247ab098a94ddc58fc96ca46caee7019c3',
|
||||
626000: '222c2ec50130a658ece226c420c76a8f7a15640033fd89ccaa29bd7c639cb94b',
|
||||
627000: 'de9830e9cc7537c42133153cccf891dcd422203827b1377df748ec27dbfd04a9',
|
||||
628000: '35d44ad84a3d9d548bbc8c91ca734979759e3a03e37153b650f794a0aed1b742',
|
||||
629000: 'c119f3acbd761598673b48eaf0d223e1729b8c8aaef9a715f1bc3dde561a4568',
|
||||
630000: 'ebfda8a47983ced56d40e95ce3be793305899cb92359ee2f9a83f7f6a59a4fe7',
|
||||
631000: '1479c2e6217b97e98f44bbb315ca99d1ab02aad14f52d60f7e6670f5c6d0fbe4',
|
||||
632000: '6ed7f9c90ea95825b1aae91644a68e36b6452e61f05a2821e94399823ad669e3',
|
||||
633000: 'e00eb0bffcf784f9f8fef21b92b9cd98e44db5d790b6eab309d0b11adf910820',
|
||||
634000: '3c71674dab2c2644d910acff4336fb02c2dd8281e30e5c97b9378d50f3d6eac0',
|
||||
635000: 'ee3fbedd791f31de2f820cf37cf5a7bcf1fddff354c1ab5736384f41846894f5',
|
||||
636000: '551b7598f5576c577b48ac7530e56422a5fd31a3860b37994f8489308830e9f6',
|
||||
637000: '531b1a767734f1750077ec9d1ab57fd4430e0154deca2fdb9dc5cfcbe980726c',
|
||||
638000: '3ae3a009b3822211b3bbe9745f006eaf3aeae365e81890205e0223925461af81',
|
||||
639000: '2aa1de8c824fcc1a8f34269010405a0a0fb921fa0f8b998010e4076fd722948c',
|
||||
640000: 'badc624e2ffabc438b7b2c9894e2134e36c2e80af700cebdf0980169c1cfe49a',
|
||||
641000: '0b550e8f9b66ac3a3d0e5a911819643600dec121959fe2b5128cc6c68503e655',
|
||||
642000: 'bfccbe9a0a127d4cf606adb5eb42f3f0ee2ebc780b719d0ec612e590aa7ec47f',
|
||||
643000: '4d44b13390410caf86d145103471469d0b10450166f5b96b8886a440aab5b1cf',
|
||||
644000: '2670950e311d8230e8d3447e97652cb5592248b59968bedb6a98a98c12b8d81d',
|
||||
645000: '6703a25f54614ff79c954438ab9fae5baff32c81fa47306b337f0f87bb11f949',
|
||||
646000: 'c93876f02c788041b400aa42149c176e624eef01bc66b49adf8fcbe255dd1833',
|
||||
647000: '276521b16f6a98f5399a9754621fbd7999b6b94e3cff17cd88d4a1176f431353',
|
||||
648000: '65b6238c589baec511c240cc87cb6ce758ad6cf119f465838a976a853c7f5be6',
|
||||
649000: 'c94f6f7fa7a8afdbecc7c291eab2a3cf218ff1af5117ed1f3f0e60214ec7041d',
|
||||
650000: 'b823008cdb0176fa0522c26b35962b4325fd526217f7eb19464cffcb3c24b4d9',
|
||||
651000: '77024b0bfd3cc7c8d52eff6db4dcec041f8675e6f3c413b7d581bedb1b53bb11',
|
||||
652000: '22a677eef4cfcee12756492eab7e184bda054481d4fc19cc46bc7147c097d118',
|
||||
653000: '66779d92b9fc28fddc46ed14b79b26b3ad7672f0dbc7cf4462d5a49769a89470',
|
||||
654000: '097eb77aede84c35822b3700673efd0129de247cb8ba5ef6b8e6dd7e85bb908b',
|
||||
655000: '83b5b4d4aeef3782c82b9a2fd53937e4523ea15867780e4264688e860fd99593',
|
||||
656000: '99646218016eda7976b45fbb6e891fc4bc130bfb3b5e6dcac8b11e0a4b2f59bd',
|
||||
657000: '9ad1995af3f7f6839a11bf5fcd856f471b200911e1e5a647390a7aa26abc6825',
|
||||
658000: 'fd5a4462737448dcfa9d81802d16f4621ea0b177f538144d836829a536cdd451',
|
||||
659000: '7f6baf86cafbdc820437d63260fcc434694dcf0ab8000307f134fc9c50f437b6',
|
||||
660000: '54a5f2c5a1f1534f23f54c5ac6e3f794ebb62f298dc9ed1aba4112e10cc778cb',
|
||||
661000: '91b46c2d247a43ea916adad7edcc37e18d149fe3ab970eaf39cd060a9c856ce2',
|
||||
662000: 'b49e0df59bfc9dcd32cd1a4b903a12accda70054fbd5f4bc480de0367f254c21',
|
||||
663000: '29d2d3e06f744ec2225659199bd43e446308fe1e1ff16c26e0e89caa468450f1',
|
||||
664000: '50b0665a8c7a9fd1ffb165fbb0148115f83fb7a5f6e9b8ad16c9dff9175d11c8',
|
||||
665000: '35754b82653110c6cb0f9a0e7011ed1ff8b360301e4280e6aa2f6faa5950a369',
|
||||
666000: 'fbc64efa658b00bbb70f5db8265e99465a07627d663c9a716ec8b42bed82338e',
|
||||
667000: 'a046d6957f24ec9f66c73612d3eb483c65e4216fc2761260d9818486cbebe56e',
|
||||
668000: '5d78ef63ea8ce0f66032368effcd56fa443b0f5c488ef8c243d3622113087fbe',
|
||||
669000: '0b1b551500e2c8617150a18deec1e3b9594dd98fc9b97e20dacf2f059f966692',
|
||||
670000: '5fa21e8edbdbb2ccccc2e3aca8e15b0965cd307ea54b6621760e654b53488d33',
|
||||
671000: '90affcb451a786aa42ff93dd1f39961a15cc8c332613f8dc591d9f9c34d359e6',
|
||||
672000: '1a8996b27fb79d43627caf0166f5c8ec72e1e0d09e6c0e17a0d9418d37719afc',
|
||||
673000: '491ec5299fa141a82b438ef5d2429eb5560a0f04412878165c86f9fbb94198af',
|
||||
674000: 'dd680d4c1101a98b6465ac85a63ef12f3ebb1eb88277d37aa341a5b387e2f1b6',
|
||||
675000: 'af8acc98b0fd9ff347acb01801f246c63c6698aff0a0cd69b0f91e17da3742df',
|
||||
676000: 'af24a5759888314e5ef719ee23b9f948b3962ac0b949231a540d45eedaa614f8',
|
||||
677000: '7ec568d4e5ceff99c0e9cefa6407784aa0246845420812bcfd8a5d9c03cb01a8',
|
||||
678000: '845df5d7e1d6f22242ffc4c976937da67ce0880a94c5bf189195d5728f208976',
|
||||
679000: '41f895ce11bd91e09ea8b16102c8f2192cdd2c754e2cb04d33edea688fa7c3b8',
|
||||
680000: 'cbf1e3a313abbe05d90ff1de6d861cf58da50b7f8f7ba43a593a3b6192d65f8a',
|
||||
681000: '1ac184be2b5edfdcc0a73a6a93d3f59a5197bd5bba2ddb1ce737251526998bc9',
|
||||
682000: '0e793523ba0a679dce1e9cf1d1189c5311e27ae8d88c35bce36ddb3c16dc34c4',
|
||||
683000: '08e07172a6b6c3d70c4379b206ed796ea8d916d314d0b6f02539becbb90077d5',
|
||||
684000: '0b67663fc37940b4d40ba88b8b7610776e35f070e4995c5ab09ccd4b86dc1143',
|
||||
685000: '8ae92af68bcc012326889b5ef89b899aa38f65dfd3d9f7dd5b29dba0c5fbebbd',
|
||||
686000: '5223ea0cffd3c59fa6597208425ab55b3a76366fbd27d22508cd208ebf2a2eec',
|
||||
687000: 'e160e55af7fd110905364f980543fca62a123c84c81e9fd38a0aeeebc30a501a',
|
||||
688000: 'fcf3b5e0afae8d665cd6f63dacd2a861aecac20f8b73a682f79cad9523c5c4f3',
|
||||
689000: '772ffab484f07ad91ed45cf6f569eff7440f4243e16e07698e4b7bd4a109e6e3',
|
||||
690000: '5b2a4fc69617ccf1787ee40a0f6d7e0af783bcd856c2c8e0ab747a91a7c68d19',
|
||||
691000: 'e4ba352a759e2425d508a4d5b58595e6ff5eb912d8bfece5a0bc646f61e77084',
|
||||
692000: 'fca2c5b6a721db85278ab55b5e2e39a445e5b1ee5ba69a17044623a6b945d0b2',
|
||||
693000: '99f4365eee70f86499ec26c373922d389cbc5e2a198e96af5d5823ad241748a1',
|
||||
694000: '0e19e76398e65cb01517dcd4ee702c9c04c0fd53cf9468ee539094aa6248e1c8',
|
||||
695000: '5addd92022007d81ced43595458e2eee2903227063af8e9edd75cccbe559930d',
|
||||
696000: 'ba25dfe3467cc9999eb8593d955032475b777cefc8006851f692266ebb83d140',
|
||||
697000: 'e10c1c734038198a99bf970ae89e3a56b2058612e35eac8141d48e147d2d47e0',
|
||||
698000: '472c4de8e57737ca48c3bdbf3c35c37f24a179f5dfa48af89efda3c3d33c131f',
|
||||
699000: 'b359f2e61a3ca3cf8b613045617e38b06767ccff72129f13971faf26a4d08234',
|
||||
700000: 'd0555d53358978ae0abcb09f911c1b3e7b5a282b4ff6d455ca9ad04299666dc5',
|
||||
701000: '5c763b8b4329809553fb58ab279ebbc6639695c45760f0c181168d41da95e6d7',
|
||||
702000: '8a6abdca484fbdac5fd6fb279a435377d964d43b62393b0d5b0cd3baaab1d2c3',
|
||||
703000: '50cf159af75b28c7a95ca990e480ebcd534c4601b03d7c0a979f26f4e33cecd7',
|
||||
704000: '0daf7f9c7eca5c6d91de9ebadb3ef1287c290700bab0aa9f6a2ac4bf42a8a98c',
|
||||
705000: 'a359e7584112eadd7fad5d961f1dee180481852d7706701c2211a98a009c128d',
|
||||
706000: '139ab9432b0c7f62739f818b114d57a59cbf20584230f1198c35f1da62c2ed62',
|
||||
707000: '43e1f4c83d26d419a35adc4dc5bd85c6253ec0faf7e70196bc5d5fac18c51746',
|
||||
708000: '1a05dba8d5d2ef4984c79cc939148ad71535043640fd5644141cdbc512623514',
|
||||
709000: 'fe7a8bd03f742de5227e5d0c8dcfd0b5cc13582703b74b1898c506fc6ffdab04',
|
||||
710000: '26217b4ecadd39e9a14ffb6cbcaab6ee7bf0954efcbbbec4de9bddb461819084',
|
||||
711000: '94aee5cc00e862e4952e573908449f4a48b630285f2f2fb20765d01dfcb68ac6',
|
||||
712000: '21fb256c0f5133634b94f6522cb8d2b0a9b982f1672ba561033be689410e7860',
|
||||
713000: '8d7d7ee4cb0598fecd85c8004b59074bedc3219f2376966e01a7faa92377cdcf',
|
||||
714000: '56a033531fc8dbb7ffe07d7365cce20281e522c3ff9fe7a19d0188af351a5799',
|
||||
715000: '4a6671846cccdc26d6fd1d77db50772315f932c24dc706004e11de68ad1ac387',
|
||||
716000: '4222447b25305bc063b668c0e010b16f9c0802fff9371e9d096ba2174926f073',
|
||||
717000: 'c231fcf5238a34cd15ac735d27b3dd0ee025714e48c408a798f0ba74be0aef77',
|
||||
718000: '0f96aaa30d20fc572d9db26871143a657b706c35a9668d31e3b64280a049787a',
|
||||
719000: '8557c088c4fc4745674c795d4a58aeb3df0dc91bb5ff93cc6268c4a88c64db5a',
|
||||
720000: 'd699e06ee7835c9d687853187125f27145ec91daf394fdee37218b8c34fee9f4',
|
||||
721000: 'c675e891a36425627e20de36f1fbdd5baba7114661f1c45f81f66a3fc55da902',
|
||||
722000: '0c6fedfb6d6c1a77254904fdd2400dedce7d45bdc2271beb77e2087a0ba30d1a',
|
||||
723000: 'a442c320886beccb3d7ea13276dbef8e98e1a47686cba2cdbeb6a2d2883af928',
|
||||
724000: '1592e2d6ac3be7535b44db6cc99080d00b19c6663f52eef3c28eae3dac27ba49',
|
||||
725000: '45b2a800f17b8571172a2658577dbe95b91ae88e611a91ac0c92609b3600f693',
|
||||
726000: 'ed6257a6567665747aa354e93ab7d3e6539d6dd41fced8a2f62cf848e2b30ce0',
|
||||
727000: '4b1a577c6c2358b0344bb1befd9b8e5572b787ec2bdc0bdcd4a150f26b2e2ab7',
|
||||
728000: '448765fbdf6261c376120ff9401db8a8841fcbed466365f982d3bc53775b93ca',
|
||||
729000: '99c2acea0af193d2e10498acd1c6d162d2a804a69157af46817b5ece5ea86491',
|
||||
730000: '94cec967e44f850f512d4240cb8a52ffaf953d0364b0a1dd7604b4a01406e669',
|
||||
731000: '6e63f5019439bc7e27a17a189baad0da8f5724883af3ca35efa0d4e5aaa75b97',
|
||||
732000: '53e1b373805f3236c7725415e872d5635b8679894c4fb630c62b6b75b4ec9d9c',
|
||||
733000: '43e9ab6cf54fde5dcdc4c473af26b256435f4af4254d96fa728f2af9b078d630',
|
||||
734000: 'a3ef7f9257d591c7dcc0f82346cb162a768ee5fe1228353ec485e69be1bf585f',
|
||||
735000: '9bc81abb6c9294463d7fa12b9ceea4f929a5491cf4b6ff8e47e0a95b02c6d355',
|
||||
736000: 'a3b391ecba546ebbbe6e05c5222beca269e5dce6e508028ea41725fef138b687',
|
||||
737000: '0f2e4e43c76b3bf6fc6db9b87adb9a17a05e85110dcb923442746a00446e513a',
|
||||
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
|
||||
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
|
||||
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
import struct
|
||||
import binascii
|
||||
from lbry.crypto.hash import double_sha256
|
||||
|
||||
|
||||
class InvalidProofError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def get_hash_for_outpoint(txhash, nout, height_of_last_takeover):
|
||||
return double_sha256(
|
||||
double_sha256(txhash) +
|
||||
double_sha256(str(nout).encode()) +
|
||||
double_sha256(struct.pack('>Q', height_of_last_takeover))
|
||||
)
|
||||
|
||||
|
||||
# noinspection PyPep8
|
||||
def verify_proof(proof, root_hash, name):
|
||||
previous_computed_hash = None
|
||||
reverse_computed_name = ''
|
||||
verified_value = False
|
||||
for i, node in enumerate(proof['nodes'][::-1]):
|
||||
found_child_in_chain = False
|
||||
to_hash = b''
|
||||
previous_child_character = None
|
||||
for child in node['children']:
|
||||
if child['character'] < 0 or child['character'] > 255:
|
||||
raise InvalidProofError("child character not int between 0 and 255")
|
||||
if previous_child_character:
|
||||
if previous_child_character >= child['character']:
|
||||
raise InvalidProofError("children not in increasing order")
|
||||
previous_child_character = child['character']
|
||||
to_hash += bytes((child['character'],))
|
||||
if 'nodeHash' in child:
|
||||
if len(child['nodeHash']) != 64:
|
||||
raise InvalidProofError("invalid child nodeHash")
|
||||
to_hash += binascii.unhexlify(child['nodeHash'])[::-1]
|
||||
else:
|
||||
if previous_computed_hash is None:
|
||||
raise InvalidProofError("previous computed hash is None")
|
||||
if found_child_in_chain is True:
|
||||
raise InvalidProofError("already found the next child in the chain")
|
||||
found_child_in_chain = True
|
||||
reverse_computed_name += chr(child['character'])
|
||||
to_hash += previous_computed_hash
|
||||
|
||||
if not found_child_in_chain:
|
||||
if i != 0:
|
||||
raise InvalidProofError("did not find the alleged child")
|
||||
if i == 0 and 'txhash' in proof and 'nOut' in proof and 'last takeover height' in proof:
|
||||
if len(proof['txhash']) != 64:
|
||||
raise InvalidProofError(f"txhash was invalid: {proof['txhash']}")
|
||||
if not isinstance(proof['nOut'], int):
|
||||
raise InvalidProofError(f"nOut was invalid: {proof['nOut']}")
|
||||
if not isinstance(proof['last takeover height'], int):
|
||||
raise InvalidProofError(
|
||||
f"last takeover height was invalid: {proof['last takeover height']}")
|
||||
to_hash += get_hash_for_outpoint(
|
||||
binascii.unhexlify(proof['txhash'])[::-1],
|
||||
proof['nOut'],
|
||||
proof['last takeover height']
|
||||
)
|
||||
verified_value = True
|
||||
elif 'valueHash' in node:
|
||||
if len(node['valueHash']) != 64:
|
||||
raise InvalidProofError("valueHash was invalid")
|
||||
to_hash += binascii.unhexlify(node['valueHash'])[::-1]
|
||||
|
||||
previous_computed_hash = double_sha256(to_hash)
|
||||
|
||||
if previous_computed_hash != binascii.unhexlify(root_hash)[::-1]:
|
||||
raise InvalidProofError("computed hash does not match roothash")
|
||||
if 'txhash' in proof and 'nOut' in proof:
|
||||
if not verified_value:
|
||||
raise InvalidProofError("mismatch between proof claim and outcome")
|
||||
target = reverse_computed_name[::-1].encode('ISO-8859-1').decode()
|
||||
if 'txhash' in proof and 'nOut' in proof:
|
||||
if name != target:
|
||||
raise InvalidProofError("name did not match proof")
|
||||
if not name.startswith(target):
|
||||
raise InvalidProofError("name fragment does not match proof")
|
||||
return True
|
||||
@@ -1,245 +0,0 @@
|
||||
import os.path
|
||||
import asyncio
|
||||
import sqlite3
|
||||
from typing import List, Optional
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from lbry.schema.url import normalize_name
|
||||
|
||||
from .bcd_data_stream import BCDataStream
|
||||
|
||||
|
||||
FILES = [
|
||||
'claims',
|
||||
'block_index',
|
||||
]
|
||||
|
||||
|
||||
def make_short_url(r):
|
||||
try:
|
||||
# TODO: we describe it as normalized but the old SDK didnt do that
|
||||
name = r["name"].decode().replace("\x00", "")
|
||||
return f'{name}#{r["shortestID"] or r["claimID"][::-1].hex()[0]}'
|
||||
except UnicodeDecodeError:
|
||||
# print(f'failed making short url due to name parse error for claim_id: {r["claimID"][::-1].hex()}')
|
||||
return "INVALID NAME"
|
||||
|
||||
|
||||
class FindShortestID:
|
||||
__slots__ = 'short_id', 'new_id'
|
||||
|
||||
def __init__(self):
|
||||
self.short_id = ''
|
||||
self.new_id = None
|
||||
|
||||
def step(self, other_id, new_id):
|
||||
other_id = other_id[::-1].hex()
|
||||
if self.new_id is None:
|
||||
self.new_id = new_id[::-1].hex()
|
||||
for i in range(len(self.new_id)):
|
||||
if other_id[i] != self.new_id[i]:
|
||||
if i > len(self.short_id)-1:
|
||||
self.short_id = self.new_id[:i+1]
|
||||
break
|
||||
|
||||
def finalize(self):
|
||||
return self.short_id
|
||||
|
||||
|
||||
class BlockchainDB:
|
||||
|
||||
def __init__(self, directory: str):
|
||||
self.directory = directory
|
||||
self.connection: Optional[sqlite3.Connection] = None
|
||||
self.executor: Optional[ThreadPoolExecutor] = None
|
||||
|
||||
async def run_in_executor(self, *args):
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, *args)
|
||||
|
||||
def sync_open(self):
|
||||
self.connection = sqlite3.connect(
|
||||
os.path.join(self.directory, FILES[0]+'.sqlite'),
|
||||
timeout=60.0 * 5
|
||||
)
|
||||
for file in FILES[1:]:
|
||||
self.connection.execute(
|
||||
f"ATTACH DATABASE '{os.path.join(self.directory, file+'.sqlite')}' AS {file}"
|
||||
)
|
||||
self.connection.create_aggregate("find_shortest_id", 2, FindShortestID)
|
||||
self.connection.execute("CREATE INDEX IF NOT EXISTS claim_originalheight ON claim (originalheight);")
|
||||
self.connection.execute("CREATE INDEX IF NOT EXISTS claim_updateheight ON claim (updateheight);")
|
||||
self.connection.execute("create index IF NOT EXISTS support_blockheight on support (blockheight);")
|
||||
self.connection.row_factory = sqlite3.Row
|
||||
|
||||
async def open(self):
|
||||
assert self.executor is None, "Database is already open."
|
||||
self.executor = ThreadPoolExecutor(max_workers=1)
|
||||
return await self.run_in_executor(self.sync_open)
|
||||
|
||||
def sync_close(self):
|
||||
self.connection.close()
|
||||
self.connection = None
|
||||
|
||||
async def close(self):
|
||||
if self.executor is not None:
|
||||
if self.connection is not None:
|
||||
await self.run_in_executor(self.sync_close)
|
||||
self.executor.shutdown()
|
||||
self.executor = None
|
||||
|
||||
async def commit(self):
|
||||
await self.run_in_executor(self.connection.commit)
|
||||
|
||||
def sync_execute(self, sql: str, *args):
|
||||
return self.connection.execute(sql, *args)
|
||||
|
||||
async def execute(self, sql: str, *args):
|
||||
return await self.run_in_executor(self.sync_execute, sql, *args)
|
||||
|
||||
def sync_execute_fetchall(self, sql: str, *args) -> List[dict]:
|
||||
return self.connection.execute(sql, *args).fetchall()
|
||||
|
||||
async def execute_fetchall(self, sql: str, *args) -> List[dict]:
|
||||
return await self.run_in_executor(self.sync_execute_fetchall, sql, *args)
|
||||
|
||||
def sync_get_best_height(self) -> int:
|
||||
sql = "SELECT MAX(height) FROM block_info"
|
||||
return self.connection.execute(sql).fetchone()[0]
|
||||
|
||||
async def get_best_height(self) -> int:
|
||||
return await self.run_in_executor(self.sync_get_best_height)
|
||||
|
||||
def sync_get_block_files(self, file_number: int = None, start_height: int = None) -> List[dict]:
|
||||
sql = """
|
||||
SELECT
|
||||
file as file_number,
|
||||
COUNT(hash) as blocks,
|
||||
SUM(txcount) as txs,
|
||||
MAX(height) as best_height,
|
||||
MIN(height) as start_height
|
||||
FROM block_info
|
||||
WHERE status&1 AND status&4 AND NOT status&32 AND NOT status&64
|
||||
"""
|
||||
args = ()
|
||||
if file_number is not None and start_height is not None:
|
||||
sql += "AND file = ? AND height >= ?"
|
||||
args = (file_number, start_height)
|
||||
return [dict(r) for r in self.sync_execute_fetchall(sql + " GROUP BY file ORDER BY file ASC;", args)]
|
||||
|
||||
async def get_block_files(self, file_number: int = None, start_height: int = None) -> List[dict]:
|
||||
return await self.run_in_executor(
|
||||
self.sync_get_block_files, file_number, start_height
|
||||
)
|
||||
|
||||
def sync_get_blocks_in_file(self, block_file: int, start_height=0) -> List[dict]:
|
||||
return [dict(r) for r in self.sync_execute_fetchall(
|
||||
"""
|
||||
SELECT datapos as data_offset, height, hash as block_hash, txCount as txs
|
||||
FROM block_info
|
||||
WHERE file = ? AND height >= ?
|
||||
AND status&1 AND status&4 AND NOT status&32 AND NOT status&64
|
||||
ORDER BY datapos ASC;
|
||||
""", (block_file, start_height)
|
||||
)]
|
||||
|
||||
async def get_blocks_in_file(self, block_file: int, start_height=0) -> List[dict]:
|
||||
return await self.run_in_executor(self.sync_get_blocks_in_file, block_file, start_height)
|
||||
|
||||
def sync_get_claim_support_txo_hashes(self, at_height: int) -> set:
|
||||
return {
|
||||
r['txID'] + BCDataStream.uint32.pack(r['txN'])
|
||||
for r in self.connection.execute(
|
||||
"""
|
||||
SELECT txID, txN FROM claim WHERE updateHeight = ?
|
||||
UNION
|
||||
SELECT txID, txN FROM support WHERE blockHeight = ?
|
||||
""", (at_height, at_height)
|
||||
).fetchall()
|
||||
}
|
||||
|
||||
def sync_get_takeover_count(self, start_height: int, end_height: int) -> int:
|
||||
sql = """
|
||||
SELECT COUNT(*) FROM claim WHERE name IN (
|
||||
SELECT name FROM takeover
|
||||
WHERE name IS NOT NULL AND height BETWEEN ? AND ?
|
||||
)
|
||||
""", (start_height, end_height)
|
||||
return self.connection.execute(*sql).fetchone()[0]
|
||||
|
||||
async def get_takeover_count(self, start_height: int, end_height: int) -> int:
|
||||
return await self.run_in_executor(self.sync_get_takeover_count, start_height, end_height)
|
||||
|
||||
def sync_get_takeovers(self, start_height: int, end_height: int) -> List[dict]:
|
||||
sql = """
|
||||
SELECT name, claimID, MAX(height) AS height FROM takeover
|
||||
WHERE name IS NOT NULL AND height BETWEEN ? AND ?
|
||||
GROUP BY name
|
||||
""", (start_height, end_height)
|
||||
return [{
|
||||
'normalized': normalize_name(r['name'].decode()),
|
||||
'claim_hash': r['claimID'],
|
||||
'height': r['height']
|
||||
} for r in self.sync_execute_fetchall(*sql)]
|
||||
|
||||
async def get_takeovers(self, start_height: int, end_height: int) -> List[dict]:
|
||||
return await self.run_in_executor(self.sync_get_takeovers, start_height, end_height)
|
||||
|
||||
def sync_get_claim_metadata_count(self, start_height: int, end_height: int) -> int:
|
||||
sql = "SELECT COUNT(*) FROM claim WHERE originalHeight BETWEEN ? AND ?"
|
||||
return self.connection.execute(sql, (start_height, end_height)).fetchone()[0]
|
||||
|
||||
async def get_claim_metadata_count(self, start_height: int, end_height: int) -> int:
|
||||
return await self.run_in_executor(self.sync_get_claim_metadata_count, start_height, end_height)
|
||||
|
||||
def sync_get_claim_metadata(self, claim_hashes) -> List[dict]:
|
||||
sql = f"""
|
||||
SELECT
|
||||
name, claimID, activationHeight, expirationHeight, originalHeight,
|
||||
(SELECT
|
||||
CASE WHEN takeover.claimID = claim.claimID THEN takeover.height END
|
||||
FROM takeover WHERE takeover.name = claim.nodename
|
||||
ORDER BY height DESC LIMIT 1
|
||||
) AS takeoverHeight,
|
||||
(SELECT find_shortest_id(c.claimid, claim.claimid) FROM claim AS c
|
||||
WHERE
|
||||
c.nodename = claim.nodename AND
|
||||
c.originalheight <= claim.originalheight AND
|
||||
c.claimid != claim.claimid
|
||||
) AS shortestID
|
||||
FROM claim
|
||||
WHERE claimID IN ({','.join(['?' for _ in claim_hashes])})
|
||||
ORDER BY claimID
|
||||
""", claim_hashes
|
||||
return [{
|
||||
"name": r["name"],
|
||||
"claim_hash": r["claimID"],
|
||||
"activation_height": r["activationHeight"],
|
||||
"expiration_height": r["expirationHeight"],
|
||||
"takeover_height": r["takeoverHeight"],
|
||||
"creation_height": r["originalHeight"],
|
||||
"short_url": make_short_url(r),
|
||||
} for r in self.sync_execute_fetchall(*sql)]
|
||||
|
||||
async def get_claim_metadata(self, start_height: int, end_height: int) -> List[dict]:
|
||||
return await self.run_in_executor(self.sync_get_claim_metadata, start_height, end_height)
|
||||
|
||||
def sync_get_support_metadata_count(self, start_height: int, end_height: int) -> int:
|
||||
sql = "SELECT COUNT(*) FROM support WHERE blockHeight BETWEEN ? AND ?"
|
||||
return self.connection.execute(sql, (start_height, end_height)).fetchone()[0]
|
||||
|
||||
async def get_support_metadata_count(self, start_height: int, end_height: int) -> int:
|
||||
return await self.run_in_executor(self.sync_get_support_metadata_count, start_height, end_height)
|
||||
|
||||
def sync_get_support_metadata(self, start_height: int, end_height: int) -> List[dict]:
|
||||
sql = """
|
||||
SELECT name, txid, txn, activationHeight, expirationHeight
|
||||
FROM support WHERE blockHeight BETWEEN ? AND ?
|
||||
""", (start_height, end_height)
|
||||
return [{
|
||||
"name": r['name'],
|
||||
"txo_hash_pk": r['txID'] + BCDataStream.uint32.pack(r['txN']),
|
||||
"activation_height": r['activationHeight'],
|
||||
"expiration_height": r['expirationHeight'],
|
||||
} for r in self.sync_execute_fetchall(*sql)]
|
||||
|
||||
async def get_support_metadata(self, start_height: int, end_height: int) -> List[dict]:
|
||||
return await self.run_in_executor(self.sync_get_support_metadata, start_height, end_height)
|
||||
@@ -1,58 +0,0 @@
|
||||
import re
|
||||
import textwrap
|
||||
from decimal import Decimal
|
||||
|
||||
from lbry.constants import COIN
|
||||
|
||||
|
||||
def lbc_to_dewies(lbc: str) -> int:
|
||||
try:
|
||||
if not isinstance(lbc, str):
|
||||
raise ValueError("{coins} must be a string")
|
||||
result = re.search(r'^(\d{1,10})\.(\d{1,8})$', lbc)
|
||||
if result is not None:
|
||||
whole, fractional = result.groups()
|
||||
return int(whole + fractional.ljust(8, "0"))
|
||||
raise ValueError(f"'{lbc}' is not a valid coin decimal")
|
||||
except ValueError:
|
||||
raise ValueError(textwrap.dedent(
|
||||
f"""
|
||||
Decimal inputs require a value in the ones place and in the tenths place
|
||||
separated by a period. The value provided, '{lbc}', is not of the correct
|
||||
format.
|
||||
|
||||
The following are examples of valid decimal inputs:
|
||||
|
||||
1.0
|
||||
0.001
|
||||
2.34500
|
||||
4534.4
|
||||
2323434.0000
|
||||
|
||||
The following are NOT valid:
|
||||
|
||||
83
|
||||
.456
|
||||
123.
|
||||
"""
|
||||
))
|
||||
|
||||
|
||||
def dewies_to_lbc(dewies) -> str:
|
||||
coins = '{:.8f}'.format(dewies / COIN).rstrip('0')
|
||||
if coins.endswith('.'):
|
||||
return coins+'0'
|
||||
else:
|
||||
return coins
|
||||
|
||||
|
||||
def dict_values_to_lbc(d):
|
||||
lbc_dict = {}
|
||||
for key, value in d.items():
|
||||
if isinstance(value, (int, Decimal)):
|
||||
lbc_dict[key] = dewies_to_lbc(value)
|
||||
elif isinstance(value, dict):
|
||||
lbc_dict[key] = dict_values_to_lbc(value)
|
||||
else:
|
||||
lbc_dict[key] = value
|
||||
return lbc_dict
|
||||
@@ -1,63 +0,0 @@
|
||||
from binascii import hexlify, unhexlify
|
||||
from lbry.constants import NULL_HASH32
|
||||
|
||||
|
||||
class TXRef:
|
||||
|
||||
__slots__ = '_id', '_hash'
|
||||
|
||||
def __init__(self):
|
||||
self._id = None
|
||||
self._hash = None
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self._id
|
||||
|
||||
@property
|
||||
def hash(self):
|
||||
return self._hash
|
||||
|
||||
@property
|
||||
def height(self):
|
||||
return -1
|
||||
|
||||
@property
|
||||
def is_null(self):
|
||||
return self.hash == NULL_HASH32
|
||||
|
||||
|
||||
class TXRefImmutable(TXRef):
|
||||
|
||||
__slots__ = ('_height', '_timestamp')
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._height = -1
|
||||
self._timestamp = -1
|
||||
|
||||
@classmethod
|
||||
def from_hash(cls, tx_hash: bytes, height: int, timestamp: int) -> 'TXRefImmutable':
|
||||
ref = cls()
|
||||
ref._hash = tx_hash
|
||||
ref._id = hexlify(tx_hash[::-1]).decode()
|
||||
ref._height = height
|
||||
ref._timestamp = timestamp
|
||||
return ref
|
||||
|
||||
@classmethod
|
||||
def from_id(cls, tx_id: str, height: int, timestamp: int) -> 'TXRefImmutable':
|
||||
ref = cls()
|
||||
ref._id = tx_id
|
||||
ref._hash = unhexlify(tx_id)[::-1]
|
||||
ref._height = height
|
||||
ref._timestamp = timestamp
|
||||
return ref
|
||||
|
||||
@property
|
||||
def height(self):
|
||||
return self._height
|
||||
|
||||
@property
|
||||
def timestamp(self):
|
||||
return self._timestamp
|
||||
@@ -1,370 +0,0 @@
|
||||
import base64
|
||||
import os
|
||||
import struct
|
||||
import asyncio
|
||||
import logging
|
||||
import zlib
|
||||
from datetime import date
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
from io import BytesIO
|
||||
from typing import Optional, Iterator, Tuple, Callable
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from lbry.crypto.hash import sha512, double_sha256, ripemd160
|
||||
from lbry.blockchain.util import ArithUint256
|
||||
from .checkpoints import HASHES
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InvalidHeader(Exception):
|
||||
|
||||
def __init__(self, height, message):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.height = height
|
||||
|
||||
|
||||
class Headers:
|
||||
|
||||
header_size = 112
|
||||
chunk_size = 10**16
|
||||
|
||||
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
|
||||
target_timespan = 150
|
||||
checkpoints = HASHES
|
||||
first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot
|
||||
timestamp_average_offset = 160.6855883050695 # calculated at 733447
|
||||
|
||||
validate_difficulty: bool = True
|
||||
|
||||
def __init__(self, path) -> None:
|
||||
if path == ':memory:':
|
||||
self.io = BytesIO()
|
||||
self.path = path
|
||||
self._size: Optional[int] = None
|
||||
self.chunk_getter: Optional[Callable] = None
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
self.known_missing_checkpointed_chunks = set()
|
||||
self.check_chunk_lock = asyncio.Lock()
|
||||
|
||||
async def open(self):
|
||||
if not self.executor:
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
if self.path != ':memory:':
|
||||
if not os.path.exists(self.path):
|
||||
self.io = open(self.path, 'w+b')
|
||||
else:
|
||||
self.io = open(self.path, 'r+b')
|
||||
bytes_size = self.io.seek(0, os.SEEK_END)
|
||||
self._size = bytes_size // self.header_size
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
|
||||
if bytes_size % self.header_size:
|
||||
log.warning("Reader file size doesnt match header size. Repairing, might take a while.")
|
||||
await self.repair()
|
||||
else:
|
||||
# try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok)
|
||||
await self.repair(start_height=max_checkpointed_height)
|
||||
await self.ensure_checkpointed_size()
|
||||
await self.get_all_missing_headers()
|
||||
|
||||
async def close(self):
|
||||
if self.executor:
|
||||
self.executor.shutdown()
|
||||
self.executor = None
|
||||
self.io.close()
|
||||
|
||||
@staticmethod
|
||||
def serialize(header):
|
||||
return b''.join([
|
||||
struct.pack('<I', header['version']),
|
||||
unhexlify(header['prev_block_hash'])[::-1],
|
||||
unhexlify(header['merkle_root'])[::-1],
|
||||
unhexlify(header['claim_trie_root'])[::-1],
|
||||
struct.pack('<III', header['timestamp'], header['bits'], header['nonce'])
|
||||
])
|
||||
|
||||
@staticmethod
|
||||
def deserialize(height, header):
|
||||
version, = struct.unpack('<I', header[:4])
|
||||
timestamp, bits, nonce = struct.unpack('<III', header[100:112])
|
||||
return {
|
||||
'version': version,
|
||||
'prev_block_hash': hexlify(header[4:36][::-1]),
|
||||
'merkle_root': hexlify(header[36:68][::-1]),
|
||||
'claim_trie_root': hexlify(header[68:100][::-1]),
|
||||
'timestamp': timestamp,
|
||||
'bits': bits,
|
||||
'nonce': nonce,
|
||||
'block_height': height,
|
||||
}
|
||||
|
||||
def get_next_chunk_target(self, chunk: int) -> ArithUint256:
|
||||
return ArithUint256(self.max_target)
|
||||
|
||||
def get_next_block_target(self, max_target: ArithUint256, previous: Optional[dict],
|
||||
current: Optional[dict]) -> ArithUint256:
|
||||
# https://github.com/lbryio/lbrycrd/blob/master/src/lbry.cpp
|
||||
if previous is None and current is None:
|
||||
return max_target
|
||||
if previous is None:
|
||||
previous = current
|
||||
actual_timespan = current['timestamp'] - previous['timestamp']
|
||||
modulated_timespan = self.target_timespan + int((actual_timespan - self.target_timespan) / 8)
|
||||
minimum_timespan = self.target_timespan - int(self.target_timespan / 8) # 150 - 18 = 132
|
||||
maximum_timespan = self.target_timespan + int(self.target_timespan / 2) # 150 + 75 = 225
|
||||
clamped_timespan = max(minimum_timespan, min(modulated_timespan, maximum_timespan))
|
||||
target = ArithUint256.from_compact(current['bits'])
|
||||
new_target = min(max_target, (target * clamped_timespan) / self.target_timespan)
|
||||
return new_target
|
||||
|
||||
def __len__(self) -> int:
|
||||
return self._size
|
||||
|
||||
def __bool__(self):
|
||||
return True
|
||||
|
||||
async def get(self, height) -> dict:
|
||||
if isinstance(height, slice):
|
||||
raise NotImplementedError("Slicing of header chain has not been implemented yet.")
|
||||
try:
|
||||
return self.deserialize(height, await self.get_raw_header(height))
|
||||
except struct.error:
|
||||
raise IndexError(f"failed to get {height}, at {len(self)}")
|
||||
|
||||
def estimated_timestamp(self, height):
|
||||
if height <= 0:
|
||||
return
|
||||
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
|
||||
|
||||
def estimated_date(self, height):
|
||||
return date.fromtimestamp(self.estimated_timestamp(height))
|
||||
|
||||
async def get_raw_header(self, height) -> bytes:
|
||||
if self.chunk_getter:
|
||||
await self.ensure_chunk_at(height)
|
||||
if not 0 <= height <= self.height:
|
||||
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, self._read, height)
|
||||
|
||||
def _read(self, height, count=1):
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
return self.io.read(self.header_size * count)
|
||||
|
||||
def chunk_hash(self, start, count):
|
||||
self.io.seek(start * self.header_size, os.SEEK_SET)
|
||||
return self.hash_header(self.io.read(count * self.header_size)).decode()
|
||||
|
||||
async def ensure_checkpointed_size(self):
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
|
||||
if self.height < max_checkpointed_height:
|
||||
self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000))
|
||||
|
||||
async def ensure_chunk_at(self, height):
|
||||
async with self.check_chunk_lock:
|
||||
if await self.has_header(height):
|
||||
log.debug("has header %s", height)
|
||||
return
|
||||
return await self.fetch_chunk(height)
|
||||
|
||||
async def fetch_chunk(self, height):
|
||||
log.info("on-demand fetching height %s", height)
|
||||
start = (height // 1000) * 1000
|
||||
headers = await self.chunk_getter(start) # pylint: disable=not-callable
|
||||
chunk = (
|
||||
zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
|
||||
)
|
||||
chunk_hash = self.hash_header(chunk).decode()
|
||||
if self.checkpoints.get(start) == chunk_hash:
|
||||
await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk)
|
||||
if start in self.known_missing_checkpointed_chunks:
|
||||
self.known_missing_checkpointed_chunks.remove(start)
|
||||
return
|
||||
elif start not in self.checkpoints:
|
||||
return # todo: fixme
|
||||
raise Exception(
|
||||
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
|
||||
)
|
||||
|
||||
async def has_header(self, height):
|
||||
normalized_height = (height // 1000) * 1000
|
||||
if normalized_height in self.checkpoints:
|
||||
return normalized_height not in self.known_missing_checkpointed_chunks
|
||||
|
||||
def _has_header(height):
|
||||
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
|
||||
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
|
||||
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, _has_header, height)
|
||||
|
||||
async def get_all_missing_headers(self):
|
||||
# Heavy operation done in one optimized shot
|
||||
def _io_checkall():
|
||||
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
|
||||
if chunk_height in self.known_missing_checkpointed_chunks:
|
||||
continue
|
||||
if self.chunk_hash(chunk_height, 1000) != expected_hash:
|
||||
self.known_missing_checkpointed_chunks.add(chunk_height)
|
||||
return self.known_missing_checkpointed_chunks
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, _io_checkall)
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
return len(self)-1
|
||||
|
||||
@property
|
||||
def bytes_size(self):
|
||||
return len(self) * self.header_size
|
||||
|
||||
async def hash(self, height=None) -> bytes:
|
||||
return self.hash_header(
|
||||
await self.get_raw_header(height if height is not None else self.height)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def hash_header(header: bytes) -> bytes:
|
||||
if header is None:
|
||||
return b'0' * 64
|
||||
return hexlify(double_sha256(header)[::-1])
|
||||
|
||||
async def connect(self, start: int, headers: bytes) -> int:
|
||||
added = 0
|
||||
bail = False
|
||||
for height, chunk in self._iterate_chunks(start, headers):
|
||||
try:
|
||||
# validate_chunk() is CPU bound and reads previous chunks from file system
|
||||
await self.validate_chunk(height, chunk)
|
||||
except InvalidHeader as e:
|
||||
bail = True
|
||||
chunk = chunk[:(height-e.height)*self.header_size]
|
||||
if chunk:
|
||||
added += await asyncio.get_running_loop().run_in_executor(self.executor, self._write, height, chunk)
|
||||
if bail:
|
||||
break
|
||||
return added
|
||||
|
||||
def _write(self, height, verified_chunk):
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
written = self.io.write(verified_chunk) // self.header_size
|
||||
# self.io.truncate()
|
||||
# .seek()/.write()/.truncate() might also .flush() when needed
|
||||
# the goal here is mainly to ensure we're definitely flush()'ing
|
||||
self.io.flush()
|
||||
self._size = max(self._size or 0, self.io.tell() // self.header_size)
|
||||
return written
|
||||
|
||||
async def validate_chunk(self, height, chunk):
|
||||
previous_hash, previous_header, previous_previous_header = None, None, None
|
||||
if height > 0:
|
||||
raw = await self.get_raw_header(height-1)
|
||||
previous_header = self.deserialize(height-1, raw)
|
||||
previous_hash = self.hash_header(raw)
|
||||
if height > 1:
|
||||
previous_previous_header = await self.get(height-2)
|
||||
chunk_target = self.get_next_chunk_target(height // 2016 - 1)
|
||||
for current_hash, current_header in self._iterate_headers(height, chunk):
|
||||
block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header)
|
||||
self.validate_header(height, current_hash, current_header, previous_hash, block_target)
|
||||
previous_previous_header = previous_header
|
||||
previous_header = current_header
|
||||
previous_hash = current_hash
|
||||
|
||||
def validate_header(self, height: int, current_hash: bytes,
|
||||
header: dict, previous_hash: bytes, target: ArithUint256):
|
||||
|
||||
if previous_hash is None:
|
||||
if self.genesis_hash is not None and self.genesis_hash != current_hash:
|
||||
raise InvalidHeader(
|
||||
height, f"genesis header doesn't match: {current_hash.decode()} "
|
||||
f"vs expected {self.genesis_hash.decode()}")
|
||||
return
|
||||
|
||||
if header['prev_block_hash'] != previous_hash:
|
||||
raise InvalidHeader(
|
||||
height, "previous hash mismatch: {} vs expected {}".format(
|
||||
header['prev_block_hash'].decode(), previous_hash.decode())
|
||||
)
|
||||
|
||||
if self.validate_difficulty:
|
||||
|
||||
if header['bits'] != target.compact:
|
||||
raise InvalidHeader(
|
||||
height, "bits mismatch: {} vs expected {}".format(
|
||||
header['bits'], target.compact)
|
||||
)
|
||||
|
||||
proof_of_work = self.get_proof_of_work(current_hash)
|
||||
if proof_of_work > target:
|
||||
raise InvalidHeader(
|
||||
height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}"
|
||||
)
|
||||
|
||||
async def repair(self, start_height=0):
|
||||
previous_header_hash = fail = None
|
||||
batch_size = 36
|
||||
for height in range(start_height, self.height, batch_size):
|
||||
headers = await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, self._read, height, batch_size
|
||||
)
|
||||
if len(headers) % self.header_size != 0:
|
||||
headers = headers[:(len(headers) // self.header_size) * self.header_size]
|
||||
for header_hash, header in self._iterate_headers(height, headers):
|
||||
height = header['block_height']
|
||||
if previous_header_hash:
|
||||
if header['prev_block_hash'] != previous_header_hash:
|
||||
fail = True
|
||||
elif height == 0:
|
||||
if header_hash != self.genesis_hash:
|
||||
fail = True
|
||||
else:
|
||||
# for sanity and clarity, since it is the only way we can end up here
|
||||
assert start_height > 0 and height == start_height
|
||||
if fail:
|
||||
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
||||
def __truncate(at_height):
|
||||
self.io.seek(max(0, (at_height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||
return await asyncio.get_running_loop().run_in_executor(self.executor, __truncate, height)
|
||||
previous_header_hash = header_hash
|
||||
|
||||
@classmethod
|
||||
def get_proof_of_work(cls, header_hash: bytes):
|
||||
return ArithUint256(int(b'0x' + cls.header_hash_to_pow_hash(header_hash), 16))
|
||||
|
||||
def _iterate_chunks(self, height: int, headers: bytes) -> Iterator[Tuple[int, bytes]]:
|
||||
assert len(headers) % self.header_size == 0, f"{len(headers)} {len(headers)%self.header_size}"
|
||||
start = 0
|
||||
end = (self.chunk_size - height % self.chunk_size) * self.header_size
|
||||
while start < end:
|
||||
yield height + (start // self.header_size), headers[start:end]
|
||||
start = end
|
||||
end = min(len(headers), end + self.chunk_size * self.header_size)
|
||||
|
||||
def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]:
|
||||
assert len(headers) % self.header_size == 0, len(headers)
|
||||
for idx in range(len(headers) // self.header_size):
|
||||
start, end = idx * self.header_size, (idx + 1) * self.header_size
|
||||
header = headers[start:end]
|
||||
yield self.hash_header(header), self.deserialize(height+idx, header)
|
||||
|
||||
@staticmethod
|
||||
def header_hash_to_pow_hash(header_hash: bytes):
|
||||
header_hash_bytes = unhexlify(header_hash)[::-1]
|
||||
h = sha512(header_hash_bytes)
|
||||
pow_hash = double_sha256(
|
||||
ripemd160(h[:len(h) // 2]) +
|
||||
ripemd160(h[len(h) // 2:])
|
||||
)
|
||||
return hexlify(pow_hash[::-1])
|
||||
|
||||
|
||||
class UnvalidatedHeaders(Headers):
|
||||
validate_difficulty = False
|
||||
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
|
||||
checkpoints = {}
|
||||
@@ -1,346 +0,0 @@
|
||||
import os
|
||||
import struct
|
||||
import shutil
|
||||
import asyncio
|
||||
import logging
|
||||
import zipfile
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from typing import Optional
|
||||
from binascii import hexlify
|
||||
|
||||
import aiohttp
|
||||
import zmq
|
||||
import zmq.asyncio
|
||||
|
||||
from lbry.conf import Config
|
||||
from lbry.event import EventController
|
||||
from lbry.error import LbrycrdEventSubscriptionError, LbrycrdUnauthorizedError
|
||||
|
||||
from .database import BlockchainDB
|
||||
from .ledger import Ledger, RegTestLedger
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DOWNLOAD_URL = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.6/lbrycrd-linux-1746.zip'
|
||||
)
|
||||
|
||||
|
||||
class Process(asyncio.SubprocessProtocol):
|
||||
|
||||
IGNORE_OUTPUT = [
|
||||
b'keypool keep',
|
||||
b'keypool reserve',
|
||||
b'keypool return',
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
self.ready = asyncio.Event()
|
||||
self.stopped = asyncio.Event()
|
||||
|
||||
def pipe_data_received(self, fd, data):
|
||||
if not any(ignore in data for ignore in self.IGNORE_OUTPUT):
|
||||
if b'Error:' in data:
|
||||
log.error(data.decode())
|
||||
else:
|
||||
for line in data.decode().splitlines():
|
||||
log.debug(line.rstrip())
|
||||
if b'Error:' in data:
|
||||
self.ready.set()
|
||||
raise SystemError(data.decode())
|
||||
if b'Done loading' in data:
|
||||
self.ready.set()
|
||||
|
||||
def process_exited(self):
|
||||
self.stopped.set()
|
||||
self.ready.set()
|
||||
|
||||
|
||||
ZMQ_BLOCK_EVENT = 'pubhashblock'
|
||||
|
||||
|
||||
class Lbrycrd:
|
||||
|
||||
def __init__(self, ledger: Ledger):
|
||||
self.ledger, self.conf = ledger, ledger.conf
|
||||
self.data_dir = self.actual_data_dir = ledger.conf.lbrycrd_dir
|
||||
if self.is_regtest:
|
||||
self.actual_data_dir = os.path.join(self.data_dir, 'regtest')
|
||||
self.blocks_dir = os.path.join(self.actual_data_dir, 'blocks')
|
||||
self.bin_dir = os.path.join(os.path.dirname(__file__), 'bin')
|
||||
self.daemon_bin = os.path.join(self.bin_dir, 'lbrycrdd')
|
||||
self.cli_bin = os.path.join(self.bin_dir, 'lbrycrd-cli')
|
||||
self.protocol = None
|
||||
self.transport = None
|
||||
self.subscribed = False
|
||||
self.subscription: Optional[asyncio.Task] = None
|
||||
self.default_generate_address = None
|
||||
self._on_block_hash_controller = EventController()
|
||||
self.on_block_hash = self._on_block_hash_controller.stream
|
||||
self.on_block_hash.listen(lambda e: log.info('%s %s', hexlify(e['hash']), e['msg']))
|
||||
self._on_tx_hash_controller = EventController()
|
||||
self.on_tx_hash = self._on_tx_hash_controller.stream
|
||||
|
||||
self.db = BlockchainDB(self.actual_data_dir)
|
||||
self._session: Optional[aiohttp.ClientSession] = None
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
|
||||
@property
|
||||
def session(self) -> aiohttp.ClientSession:
|
||||
if self._session is None:
|
||||
self._session = aiohttp.ClientSession()
|
||||
return self._session
|
||||
|
||||
@classmethod
|
||||
def temp_regtest(cls):
|
||||
return cls(RegTestLedger(
|
||||
Config.with_same_dir(tempfile.mkdtemp()).set(
|
||||
blockchain="regtest",
|
||||
lbrycrd_rpc_port=9245 + 2, # avoid conflict with default rpc port
|
||||
lbrycrd_peer_port=9246 + 2, # avoid conflict with default peer port
|
||||
lbrycrd_zmq="tcp://127.0.0.1:29002"
|
||||
)
|
||||
))
|
||||
|
||||
@staticmethod
|
||||
def get_block_file_name(block_file_number):
|
||||
return f'blk{block_file_number:05}.dat'
|
||||
|
||||
def get_block_file_path(self, block_file_number):
|
||||
return os.path.join(
|
||||
self.actual_data_dir, 'blocks',
|
||||
self.get_block_file_name(block_file_number)
|
||||
)
|
||||
|
||||
@property
|
||||
def is_regtest(self):
|
||||
return isinstance(self.ledger, RegTestLedger)
|
||||
|
||||
@property
|
||||
def rpc_url(self):
|
||||
return (
|
||||
f'http://{self.conf.lbrycrd_rpc_user}:{self.conf.lbrycrd_rpc_pass}'
|
||||
f'@{self.conf.lbrycrd_rpc_host}:{self.conf.lbrycrd_rpc_port}/'
|
||||
)
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return (
|
||||
os.path.exists(self.cli_bin) and
|
||||
os.path.exists(self.daemon_bin)
|
||||
)
|
||||
|
||||
async def download(self):
|
||||
downloaded_file = os.path.join(
|
||||
self.bin_dir, DOWNLOAD_URL[DOWNLOAD_URL.rfind('/')+1:]
|
||||
)
|
||||
|
||||
if not os.path.exists(self.bin_dir):
|
||||
os.mkdir(self.bin_dir)
|
||||
|
||||
if not os.path.exists(downloaded_file):
|
||||
log.info('Downloading: %s', DOWNLOAD_URL)
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(DOWNLOAD_URL) as response:
|
||||
with open(downloaded_file, 'wb') as out_file:
|
||||
while True:
|
||||
chunk = await response.content.read(4096)
|
||||
if not chunk:
|
||||
break
|
||||
out_file.write(chunk)
|
||||
with urllib.request.urlopen(DOWNLOAD_URL) as response:
|
||||
with open(downloaded_file, 'wb') as out_file:
|
||||
shutil.copyfileobj(response, out_file)
|
||||
|
||||
log.info('Extracting: %s', downloaded_file)
|
||||
|
||||
with zipfile.ZipFile(downloaded_file) as dotzip:
|
||||
dotzip.extractall(self.bin_dir)
|
||||
# zipfile bug https://bugs.python.org/issue15795
|
||||
os.chmod(self.cli_bin, 0o755)
|
||||
os.chmod(self.daemon_bin, 0o755)
|
||||
|
||||
return self.exists
|
||||
|
||||
async def ensure(self):
|
||||
return self.exists or await self.download()
|
||||
|
||||
def get_start_command(self, *args):
|
||||
if self.is_regtest:
|
||||
args += ('-regtest',)
|
||||
if self.conf.lbrycrd_zmq:
|
||||
args += (
|
||||
f'-zmqpubhashblock={self.conf.lbrycrd_zmq}',
|
||||
f'-zmqpubhashtx={self.conf.lbrycrd_zmq}',
|
||||
)
|
||||
return (
|
||||
self.daemon_bin,
|
||||
f'-datadir={self.data_dir}',
|
||||
f'-port={self.conf.lbrycrd_peer_port}',
|
||||
f'-rpcport={self.conf.lbrycrd_rpc_port}',
|
||||
f'-rpcuser={self.conf.lbrycrd_rpc_user}',
|
||||
f'-rpcpassword={self.conf.lbrycrd_rpc_pass}',
|
||||
'-server', '-printtoconsole',
|
||||
*args
|
||||
)
|
||||
|
||||
async def open(self):
|
||||
await self.db.open()
|
||||
|
||||
async def close(self):
|
||||
await self.db.close()
|
||||
await self.close_session()
|
||||
|
||||
async def close_session(self):
|
||||
if self._session is not None:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
|
||||
async def start(self, *args):
|
||||
loop = asyncio.get_running_loop()
|
||||
command = self.get_start_command(*args)
|
||||
log.info(' '.join(command))
|
||||
self.transport, self.protocol = await loop.subprocess_exec(Process, *command)
|
||||
await self.protocol.ready.wait()
|
||||
assert not self.protocol.stopped.is_set()
|
||||
await self.open()
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
try:
|
||||
await self.close()
|
||||
self.transport.terminate()
|
||||
await self.protocol.stopped.wait()
|
||||
assert self.transport.get_returncode() == 0, "lbrycrd daemon exit with error"
|
||||
self.transport.close()
|
||||
finally:
|
||||
if cleanup:
|
||||
await self.cleanup()
|
||||
|
||||
async def cleanup(self):
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
None, shutil.rmtree, self.data_dir, True
|
||||
)
|
||||
|
||||
async def ensure_subscribable(self):
|
||||
zmq_notifications = await self.get_zmq_notifications()
|
||||
subs = {e['type']: e['address'] for e in zmq_notifications}
|
||||
if ZMQ_BLOCK_EVENT not in subs:
|
||||
raise LbrycrdEventSubscriptionError(ZMQ_BLOCK_EVENT)
|
||||
if not self.conf.lbrycrd_zmq:
|
||||
self.conf.lbrycrd_zmq = subs[ZMQ_BLOCK_EVENT]
|
||||
|
||||
async def subscribe(self):
|
||||
if not self.subscribed:
|
||||
self.subscribed = True
|
||||
ctx = zmq.asyncio.Context.instance()
|
||||
sock = ctx.socket(zmq.SUB) # pylint: disable=no-member
|
||||
sock.connect(self.conf.lbrycrd_zmq)
|
||||
sock.subscribe("hashblock")
|
||||
sock.subscribe("hashtx")
|
||||
self.subscription = asyncio.create_task(self.subscription_handler(sock))
|
||||
|
||||
async def subscription_handler(self, sock):
|
||||
try:
|
||||
while self.subscribed:
|
||||
msg = await sock.recv_multipart()
|
||||
if msg[0] == b'hashtx':
|
||||
await self._on_tx_hash_controller.add({
|
||||
'hash': msg[1],
|
||||
'msg': struct.unpack('<I', msg[2])[0]
|
||||
})
|
||||
elif msg[0] == b'hashblock':
|
||||
await self._on_block_hash_controller.add({
|
||||
'hash': msg[1],
|
||||
'msg': struct.unpack('<I', msg[2])[0]
|
||||
})
|
||||
except asyncio.CancelledError:
|
||||
sock.close()
|
||||
raise
|
||||
|
||||
def unsubscribe(self):
|
||||
if self.subscribed:
|
||||
self.subscribed = False
|
||||
self.subscription.cancel()
|
||||
self.subscription = None
|
||||
|
||||
def sync_run(self, coro):
|
||||
if self._loop is None:
|
||||
try:
|
||||
self._loop = asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
self._loop = asyncio.new_event_loop()
|
||||
return self._loop.run_until_complete(coro)
|
||||
|
||||
async def rpc(self, method, params=None):
|
||||
if self._session is not None and self._session.closed:
|
||||
raise Exception("session is closed! RPC attempted during shutting down.")
|
||||
message = {
|
||||
"jsonrpc": "1.0",
|
||||
"id": "1",
|
||||
"method": method,
|
||||
"params": params or []
|
||||
}
|
||||
async with self.session.post(self.rpc_url, json=message) as resp:
|
||||
if resp.status == 401:
|
||||
raise LbrycrdUnauthorizedError()
|
||||
try:
|
||||
result = await resp.json()
|
||||
except aiohttp.ContentTypeError as e:
|
||||
raise Exception(await resp.text()) from e
|
||||
if not result['error']:
|
||||
return result['result']
|
||||
else:
|
||||
result['error'].update(method=method, params=params)
|
||||
raise Exception(result['error'])
|
||||
|
||||
async def get_zmq_notifications(self):
|
||||
return await self.rpc("getzmqnotifications")
|
||||
|
||||
async def generate(self, blocks):
|
||||
if self.default_generate_address is None:
|
||||
self.default_generate_address = await self.get_new_address()
|
||||
return await self.generate_to_address(blocks, self.default_generate_address)
|
||||
|
||||
async def get_new_address(self):
|
||||
return await self.rpc("getnewaddress")
|
||||
|
||||
async def generate_to_address(self, blocks, address):
|
||||
return await self.rpc("generatetoaddress", [blocks, address])
|
||||
|
||||
async def send_to_address(self, address, amount):
|
||||
return await self.rpc("sendtoaddress", [address, amount])
|
||||
|
||||
async def get_block(self, block_hash):
|
||||
return await self.rpc("getblock", [block_hash])
|
||||
|
||||
async def get_raw_mempool(self):
|
||||
return await self.rpc("getrawmempool")
|
||||
|
||||
async def get_raw_transaction(self, txid):
|
||||
return await self.rpc("getrawtransaction", [txid])
|
||||
|
||||
async def fund_raw_transaction(self, tx):
|
||||
return await self.rpc("fundrawtransaction", [tx])
|
||||
|
||||
async def sign_raw_transaction_with_wallet(self, tx):
|
||||
return await self.rpc("signrawtransactionwithwallet", [tx])
|
||||
|
||||
async def send_raw_transaction(self, tx):
|
||||
return await self.rpc("sendrawtransaction", [tx])
|
||||
|
||||
async def claim_name(self, name, data, amount):
|
||||
return await self.rpc("claimname", [name, data, amount])
|
||||
|
||||
async def update_claim(self, txid, data, amount):
|
||||
return await self.rpc("updateclaim", [txid, data, amount])
|
||||
|
||||
async def abandon_claim(self, txid, address):
|
||||
return await self.rpc("abandonclaim", [txid, address])
|
||||
|
||||
async def support_claim(self, name, claim_id, amount, value="", istip=False):
|
||||
return await self.rpc("supportclaim", [name, claim_id, amount, value, istip])
|
||||
|
||||
async def abandon_support(self, txid, address):
|
||||
return await self.rpc("abandonsupport", [txid, address])
|
||||
@@ -1,179 +0,0 @@
|
||||
from binascii import unhexlify
|
||||
from string import hexdigits
|
||||
from typing import TYPE_CHECKING, Type
|
||||
|
||||
from lbry.crypto.hash import hash160, double_sha256
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.schema.url import URL
|
||||
from .header import Headers, UnvalidatedHeaders
|
||||
from .checkpoints import HASHES
|
||||
from .dewies import lbc_to_dewies
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
|
||||
|
||||
class Ledger:
|
||||
name = 'LBRY Credits'
|
||||
symbol = 'LBC'
|
||||
network_name = 'mainnet'
|
||||
|
||||
headers_class = Headers
|
||||
|
||||
secret_prefix = bytes((0x1c,))
|
||||
pubkey_address_prefix = bytes((0x55,))
|
||||
script_address_prefix = bytes((0x7a,))
|
||||
extended_public_key_prefix = unhexlify('0488b21e')
|
||||
extended_private_key_prefix = unhexlify('0488ade4')
|
||||
|
||||
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
|
||||
genesis_bits = 0x1f00ffff
|
||||
target_timespan = 150
|
||||
|
||||
fee_per_byte = 50
|
||||
fee_per_name_char = 200000
|
||||
|
||||
checkpoints = HASHES
|
||||
|
||||
def __init__(self, conf: 'Config'):
|
||||
self.conf = conf
|
||||
self.coin_selection_strategy = None
|
||||
|
||||
@classmethod
|
||||
def get_id(cls):
|
||||
return '{}_{}'.format(cls.symbol.lower(), cls.network_name.lower())
|
||||
|
||||
@staticmethod
|
||||
def address_to_hash160(address) -> bytes:
|
||||
return Base58.decode(address)[1:21]
|
||||
|
||||
@classmethod
|
||||
def pubkey_hash_to_address(cls, h160):
|
||||
raw_address = cls.pubkey_address_prefix + h160
|
||||
return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4]))
|
||||
|
||||
@classmethod
|
||||
def public_key_to_address(cls, public_key):
|
||||
return cls.pubkey_hash_to_address(hash160(public_key))
|
||||
|
||||
@classmethod
|
||||
def script_hash_to_address(cls, h160):
|
||||
raw_address = cls.script_address_prefix + h160
|
||||
return Base58.encode(bytearray(raw_address + double_sha256(raw_address)[0:4]))
|
||||
|
||||
@staticmethod
|
||||
def private_key_to_wif(private_key):
|
||||
return b'\x1c' + private_key + b'\x01'
|
||||
|
||||
@classmethod
|
||||
def is_valid_address(cls, address):
|
||||
decoded = Base58.decode_check(address)
|
||||
return decoded[0] == cls.pubkey_address_prefix[0]
|
||||
|
||||
@classmethod
|
||||
def valid_address_or_error(cls, address):
|
||||
try:
|
||||
assert cls.is_valid_address(address)
|
||||
except:
|
||||
raise Exception(f"'{address}' is not a valid address")
|
||||
|
||||
@staticmethod
|
||||
def valid_claim_id(claim_id: str):
|
||||
if not len(claim_id) == 40:
|
||||
raise Exception(f"Incorrect claimid length: {len(claim_id)}")
|
||||
if set(claim_id).difference(hexdigits):
|
||||
raise Exception("Claim id is not hex encoded")
|
||||
|
||||
@staticmethod
|
||||
def valid_channel_name_or_error(name: str):
|
||||
try:
|
||||
if not name:
|
||||
raise Exception("Channel name cannot be blank.")
|
||||
parsed = URL.parse(name)
|
||||
if not parsed.has_channel:
|
||||
raise Exception("Channel names must start with '@' symbol.")
|
||||
if parsed.channel.name != name:
|
||||
raise Exception("Channel name has invalid character")
|
||||
except (TypeError, ValueError):
|
||||
raise Exception("Invalid channel name.")
|
||||
|
||||
@staticmethod
|
||||
def valid_stream_name_or_error(name: str):
|
||||
try:
|
||||
if not name:
|
||||
raise Exception('Stream name cannot be blank.')
|
||||
parsed = URL.parse(name)
|
||||
if parsed.has_channel:
|
||||
raise Exception(
|
||||
"Stream names cannot start with '@' symbol. This is reserved for channels claims."
|
||||
)
|
||||
if not parsed.has_stream or parsed.stream.name != name:
|
||||
raise Exception('Stream name has invalid characters.')
|
||||
except (TypeError, ValueError):
|
||||
raise Exception("Invalid stream name.")
|
||||
|
||||
@staticmethod
|
||||
def valid_collection_name_or_error(name: str):
|
||||
try:
|
||||
if not name:
|
||||
raise Exception('Collection name cannot be blank.')
|
||||
parsed = URL.parse(name)
|
||||
if parsed.has_channel:
|
||||
raise Exception(
|
||||
"Collection names cannot start with '@' symbol. This is reserved for channels claims."
|
||||
)
|
||||
if not parsed.has_stream or parsed.stream.name != name:
|
||||
raise Exception('Collection name has invalid characters.')
|
||||
except (TypeError, ValueError):
|
||||
raise Exception("Invalid collection name.")
|
||||
|
||||
@staticmethod
|
||||
def get_dewies_or_error(argument: str, lbc: str, positive_value=False):
|
||||
try:
|
||||
dewies = lbc_to_dewies(lbc)
|
||||
if positive_value and dewies <= 0:
|
||||
raise ValueError(f"'{argument}' value must be greater than 0.0")
|
||||
return dewies
|
||||
except ValueError as e:
|
||||
raise ValueError(f"Invalid value for '{argument}': {e.args[0]}")
|
||||
|
||||
def get_fee_address(self, kwargs: dict, claim_address: str) -> str:
|
||||
if 'fee_address' in kwargs:
|
||||
self.valid_address_or_error(kwargs['fee_address'])
|
||||
return kwargs['fee_address']
|
||||
if 'fee_currency' in kwargs or 'fee_amount' in kwargs:
|
||||
return claim_address
|
||||
|
||||
|
||||
class TestNetLedger(Ledger):
|
||||
network_name = 'testnet'
|
||||
pubkey_address_prefix = bytes((111,))
|
||||
script_address_prefix = bytes((196,))
|
||||
extended_public_key_prefix = unhexlify('043587cf')
|
||||
extended_private_key_prefix = unhexlify('04358394')
|
||||
checkpoints = {}
|
||||
|
||||
|
||||
class RegTestLedger(Ledger):
|
||||
network_name = 'regtest'
|
||||
headers_class = UnvalidatedHeaders
|
||||
pubkey_address_prefix = bytes((111,))
|
||||
script_address_prefix = bytes((196,))
|
||||
extended_public_key_prefix = unhexlify('043587cf')
|
||||
extended_private_key_prefix = unhexlify('04358394')
|
||||
|
||||
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash = '6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
|
||||
genesis_bits = 0x207fffff
|
||||
target_timespan = 1
|
||||
checkpoints = {}
|
||||
|
||||
|
||||
def ledger_class_from_name(name) -> Type[Ledger]:
|
||||
return {
|
||||
Ledger.network_name: Ledger,
|
||||
TestNetLedger.network_name: TestNetLedger,
|
||||
RegTestLedger.network_name: RegTestLedger
|
||||
}[name]
|
||||
@@ -1,626 +0,0 @@
|
||||
from typing import List
|
||||
from itertools import chain
|
||||
from binascii import hexlify
|
||||
from collections import namedtuple
|
||||
|
||||
from .bcd_data_stream import BCDataStream
|
||||
|
||||
|
||||
def subclass_tuple(name, base):
|
||||
return type(name, (base,), {'__slots__': ()})
|
||||
|
||||
|
||||
# bitcoin opcodes
|
||||
OP_0 = 0x00
|
||||
OP_1 = 0x51
|
||||
OP_16 = 0x60
|
||||
OP_VERIFY = 0x69
|
||||
OP_DUP = 0x76
|
||||
OP_HASH160 = 0xa9
|
||||
OP_EQUALVERIFY = 0x88
|
||||
OP_CHECKSIG = 0xac
|
||||
OP_CHECKMULTISIG = 0xae
|
||||
OP_EQUAL = 0x87
|
||||
OP_PUSHDATA1 = 0x4c
|
||||
OP_PUSHDATA2 = 0x4d
|
||||
OP_PUSHDATA4 = 0x4e
|
||||
OP_RETURN = 0x6a
|
||||
OP_2DROP = 0x6d
|
||||
OP_DROP = 0x75
|
||||
|
||||
# lbry custom opcodes
|
||||
# checks
|
||||
OP_PRICECHECK = 0xb0 # checks that the BUY output is >= SELL price
|
||||
# tx types
|
||||
OP_CLAIM_NAME = 0xb5
|
||||
OP_SUPPORT_CLAIM = 0xb6
|
||||
OP_UPDATE_CLAIM = 0xb7
|
||||
OP_SELL_CLAIM = 0xb8
|
||||
OP_BUY_CLAIM = 0xb9
|
||||
|
||||
# template matching opcodes (not real opcodes)
|
||||
# base class for PUSH_DATA related opcodes
|
||||
# pylint: disable=invalid-name
|
||||
PUSH_DATA_OP = namedtuple('PUSH_DATA_OP', 'name')
|
||||
# opcode for variable length strings
|
||||
# pylint: disable=invalid-name
|
||||
PUSH_SINGLE = subclass_tuple('PUSH_SINGLE', PUSH_DATA_OP)
|
||||
# opcode for variable size integers
|
||||
# pylint: disable=invalid-name
|
||||
PUSH_INTEGER = subclass_tuple('PUSH_INTEGER', PUSH_DATA_OP)
|
||||
# opcode for variable number of variable length strings
|
||||
# pylint: disable=invalid-name
|
||||
PUSH_MANY = subclass_tuple('PUSH_MANY', PUSH_DATA_OP)
|
||||
# opcode with embedded subscript parsing
|
||||
# pylint: disable=invalid-name
|
||||
PUSH_SUBSCRIPT = namedtuple('PUSH_SUBSCRIPT', 'name template')
|
||||
|
||||
|
||||
def is_push_data_opcode(opcode):
|
||||
return isinstance(opcode, (PUSH_DATA_OP, PUSH_SUBSCRIPT))
|
||||
|
||||
|
||||
def is_push_data_token(token):
|
||||
return 1 <= token <= OP_PUSHDATA4
|
||||
|
||||
|
||||
def push_data(data):
|
||||
size = len(data)
|
||||
if size < OP_PUSHDATA1:
|
||||
yield BCDataStream.uint8.pack(size)
|
||||
elif size <= 0xFF:
|
||||
yield BCDataStream.uint8.pack(OP_PUSHDATA1)
|
||||
yield BCDataStream.uint8.pack(size)
|
||||
elif size <= 0xFFFF:
|
||||
yield BCDataStream.uint8.pack(OP_PUSHDATA2)
|
||||
yield BCDataStream.uint16.pack(size)
|
||||
else:
|
||||
yield BCDataStream.uint8.pack(OP_PUSHDATA4)
|
||||
yield BCDataStream.uint32.pack(size)
|
||||
yield bytes(data)
|
||||
|
||||
|
||||
def read_data(token, stream):
|
||||
if token < OP_PUSHDATA1:
|
||||
return stream.read(token)
|
||||
if token == OP_PUSHDATA1:
|
||||
return stream.read(stream.read_uint8())
|
||||
if token == OP_PUSHDATA2:
|
||||
return stream.read(stream.read_uint16())
|
||||
return stream.read(stream.read_uint32())
|
||||
|
||||
|
||||
# opcode for OP_1 - OP_16
|
||||
# pylint: disable=invalid-name
|
||||
SMALL_INTEGER = namedtuple('SMALL_INTEGER', 'name')
|
||||
|
||||
|
||||
def is_small_integer(token):
|
||||
return OP_1 <= token <= OP_16
|
||||
|
||||
|
||||
def push_small_integer(num):
|
||||
assert 1 <= num <= 16
|
||||
yield BCDataStream.uint8.pack(OP_1 + (num - 1))
|
||||
|
||||
|
||||
def read_small_integer(token):
|
||||
return (token - OP_1) + 1
|
||||
|
||||
|
||||
class Token(namedtuple('Token', 'value')):
|
||||
__slots__ = ()
|
||||
|
||||
def __repr__(self):
|
||||
name = None
|
||||
for var_name, var_value in globals().items():
|
||||
if var_name.startswith('OP_') and var_value == self.value:
|
||||
name = var_name
|
||||
break
|
||||
return name or self.value
|
||||
|
||||
|
||||
class DataToken(Token):
|
||||
__slots__ = ()
|
||||
|
||||
def __repr__(self):
|
||||
return f'"{hexlify(self.value)}"'
|
||||
|
||||
|
||||
class SmallIntegerToken(Token):
|
||||
__slots__ = ()
|
||||
|
||||
def __repr__(self):
|
||||
return f'SmallIntegerToken({self.value})'
|
||||
|
||||
|
||||
def token_producer(source):
|
||||
token = source.read_uint8()
|
||||
while token is not None:
|
||||
if is_push_data_token(token):
|
||||
yield DataToken(read_data(token, source))
|
||||
elif is_small_integer(token):
|
||||
yield SmallIntegerToken(read_small_integer(token))
|
||||
else:
|
||||
yield Token(token)
|
||||
token = source.read_uint8()
|
||||
|
||||
|
||||
def tokenize(source):
|
||||
return list(token_producer(source))
|
||||
|
||||
|
||||
class ScriptError(Exception):
|
||||
""" General script handling error. """
|
||||
|
||||
|
||||
class ParseError(ScriptError):
|
||||
""" Script parsing error. """
|
||||
|
||||
|
||||
class Parser:
|
||||
|
||||
def __init__(self, opcodes, tokens):
|
||||
self.opcodes = opcodes
|
||||
self.tokens = tokens
|
||||
self.values = {}
|
||||
self.token_index = 0
|
||||
self.opcode_index = 0
|
||||
|
||||
def parse(self):
|
||||
while self.token_index < len(self.tokens) and self.opcode_index < len(self.opcodes):
|
||||
token = self.tokens[self.token_index]
|
||||
opcode = self.opcodes[self.opcode_index]
|
||||
if token.value == 0 and isinstance(opcode, PUSH_SINGLE):
|
||||
token = DataToken(b'')
|
||||
if isinstance(token, DataToken):
|
||||
if isinstance(opcode, (PUSH_SINGLE, PUSH_INTEGER, PUSH_SUBSCRIPT)):
|
||||
self.push_single(opcode, token.value)
|
||||
elif isinstance(opcode, PUSH_MANY):
|
||||
self.consume_many_non_greedy()
|
||||
else:
|
||||
raise ParseError(f"DataToken found but opcode was '{opcode}'.")
|
||||
elif isinstance(token, SmallIntegerToken):
|
||||
if isinstance(opcode, SMALL_INTEGER):
|
||||
self.values[opcode.name] = token.value
|
||||
else:
|
||||
raise ParseError(f"SmallIntegerToken found but opcode was '{opcode}'.")
|
||||
elif token.value == opcode:
|
||||
pass
|
||||
else:
|
||||
raise ParseError(f"Token is '{token.value}' and opcode is '{opcode}'.")
|
||||
self.token_index += 1
|
||||
self.opcode_index += 1
|
||||
|
||||
if self.token_index < len(self.tokens):
|
||||
raise ParseError("Parse completed without all tokens being consumed.")
|
||||
|
||||
if self.opcode_index < len(self.opcodes):
|
||||
raise ParseError("Parse completed without all opcodes being consumed.")
|
||||
|
||||
return self
|
||||
|
||||
def consume_many_non_greedy(self):
|
||||
""" Allows PUSH_MANY to consume data without being greedy
|
||||
in cases when one or more PUSH_SINGLEs follow a PUSH_MANY. This will
|
||||
prioritize giving all PUSH_SINGLEs some data and only after that
|
||||
subsume the rest into PUSH_MANY.
|
||||
"""
|
||||
|
||||
token_values = []
|
||||
while self.token_index < len(self.tokens):
|
||||
token = self.tokens[self.token_index]
|
||||
if not isinstance(token, DataToken):
|
||||
self.token_index -= 1
|
||||
break
|
||||
token_values.append(token.value)
|
||||
self.token_index += 1
|
||||
|
||||
push_opcodes = []
|
||||
push_many_count = 0
|
||||
while self.opcode_index < len(self.opcodes):
|
||||
opcode = self.opcodes[self.opcode_index]
|
||||
if not is_push_data_opcode(opcode):
|
||||
self.opcode_index -= 1
|
||||
break
|
||||
if isinstance(opcode, PUSH_MANY):
|
||||
push_many_count += 1
|
||||
push_opcodes.append(opcode)
|
||||
self.opcode_index += 1
|
||||
|
||||
if push_many_count > 1:
|
||||
raise ParseError(
|
||||
"Cannot have more than one consecutive PUSH_MANY, as there is no way to tell which"
|
||||
" token value should go into which PUSH_MANY."
|
||||
)
|
||||
|
||||
if len(push_opcodes) > len(token_values):
|
||||
raise ParseError(
|
||||
"Not enough token values to match all of the PUSH_MANY and PUSH_SINGLE opcodes."
|
||||
)
|
||||
|
||||
many_opcode = push_opcodes.pop(0)
|
||||
|
||||
# consume data into PUSH_SINGLE opcodes, working backwards
|
||||
for opcode in reversed(push_opcodes):
|
||||
self.push_single(opcode, token_values.pop())
|
||||
|
||||
# finally PUSH_MANY gets everything that's left
|
||||
self.values[many_opcode.name] = token_values
|
||||
|
||||
def push_single(self, opcode, value):
|
||||
if isinstance(opcode, PUSH_SINGLE):
|
||||
self.values[opcode.name] = value
|
||||
elif isinstance(opcode, PUSH_INTEGER):
|
||||
self.values[opcode.name] = int.from_bytes(value, 'little')
|
||||
elif isinstance(opcode, PUSH_SUBSCRIPT):
|
||||
self.values[opcode.name] = Script.from_source_with_template(value, opcode.template)
|
||||
else:
|
||||
raise ParseError(f"Not a push single or subscript: {opcode}")
|
||||
|
||||
|
||||
class Template:
|
||||
|
||||
__slots__ = 'name', 'opcodes'
|
||||
|
||||
def __init__(self, name, opcodes):
|
||||
self.name = name
|
||||
self.opcodes = opcodes
|
||||
|
||||
def parse(self, tokens):
|
||||
return Parser(self.opcodes, tokens).parse().values if self.opcodes else {}
|
||||
|
||||
def generate(self, values):
|
||||
source = BCDataStream()
|
||||
for opcode in self.opcodes:
|
||||
if isinstance(opcode, PUSH_SINGLE):
|
||||
data = values[opcode.name]
|
||||
source.write_many(push_data(data))
|
||||
elif isinstance(opcode, PUSH_INTEGER):
|
||||
data = values[opcode.name]
|
||||
source.write_many(push_data(
|
||||
data.to_bytes((data.bit_length() + 7) // 8, byteorder='little')
|
||||
))
|
||||
elif isinstance(opcode, PUSH_SUBSCRIPT):
|
||||
data = values[opcode.name]
|
||||
source.write_many(push_data(data.source))
|
||||
elif isinstance(opcode, PUSH_MANY):
|
||||
for data in values[opcode.name]:
|
||||
source.write_many(push_data(data))
|
||||
elif isinstance(opcode, SMALL_INTEGER):
|
||||
data = values[opcode.name]
|
||||
source.write_many(push_small_integer(data))
|
||||
else:
|
||||
source.write_uint8(opcode)
|
||||
return source.get_bytes()
|
||||
|
||||
|
||||
class Script:
|
||||
|
||||
__slots__ = 'source', 'offset', '_template', '_values', '_template_hint'
|
||||
|
||||
templates: List[Template] = []
|
||||
|
||||
NO_SCRIPT = Template('no_script', None) # special case
|
||||
|
||||
def __init__(self, source=None, template=None, values=None, template_hint=None, offset=None):
|
||||
self.source = source
|
||||
self.offset = offset
|
||||
self._template = template
|
||||
self._values = values
|
||||
self._template_hint = template_hint
|
||||
if source is None and template and values:
|
||||
self.generate()
|
||||
|
||||
@property
|
||||
def length(self):
|
||||
return len(self.source)
|
||||
|
||||
@property
|
||||
def template(self):
|
||||
if self._template is None:
|
||||
self.parse(self._template_hint)
|
||||
return self._template
|
||||
|
||||
@property
|
||||
def values(self):
|
||||
if self._values is None:
|
||||
self.parse(self._template_hint)
|
||||
return self._values
|
||||
|
||||
@property
|
||||
def tokens(self):
|
||||
return tokenize(BCDataStream(self.source))
|
||||
|
||||
@classmethod
|
||||
def from_source_with_template(cls, source, template):
|
||||
return cls(source, template_hint=template)
|
||||
|
||||
def parse(self, template_hint=None):
|
||||
tokens = self.tokens
|
||||
if not tokens and not template_hint:
|
||||
template_hint = self.NO_SCRIPT
|
||||
for template in chain((template_hint,), self.templates):
|
||||
if not template:
|
||||
continue
|
||||
try:
|
||||
self._values = template.parse(tokens)
|
||||
self._template = template
|
||||
return
|
||||
except ParseError:
|
||||
continue
|
||||
raise ValueError(f'No matching templates for source: {hexlify(self.source)}')
|
||||
|
||||
def generate(self):
|
||||
self.source = self.template.generate(self._values)
|
||||
|
||||
|
||||
class InputScript(Script):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
REDEEM_PUBKEY = Template('pubkey', (
|
||||
PUSH_SINGLE('signature'),
|
||||
))
|
||||
REDEEM_PUBKEY_HASH = Template('pubkey_hash', (
|
||||
PUSH_SINGLE('signature'), PUSH_SINGLE('pubkey')
|
||||
))
|
||||
REDEEM_SCRIPT = Template('script', (
|
||||
SMALL_INTEGER('signatures_count'), PUSH_MANY('pubkeys'), SMALL_INTEGER('pubkeys_count'),
|
||||
OP_CHECKMULTISIG
|
||||
))
|
||||
REDEEM_SCRIPT_HASH = Template('script_hash', (
|
||||
OP_0, PUSH_MANY('signatures'), PUSH_SUBSCRIPT('script', REDEEM_SCRIPT)
|
||||
))
|
||||
|
||||
templates = [
|
||||
REDEEM_PUBKEY,
|
||||
REDEEM_PUBKEY_HASH,
|
||||
REDEEM_SCRIPT_HASH,
|
||||
REDEEM_SCRIPT
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def redeem_pubkey_hash(cls, signature, pubkey):
|
||||
return cls(template=cls.REDEEM_PUBKEY_HASH, values={
|
||||
'signature': signature,
|
||||
'pubkey': pubkey
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def redeem_script_hash(cls, signatures, pubkeys):
|
||||
return cls(template=cls.REDEEM_SCRIPT_HASH, values={
|
||||
'signatures': signatures,
|
||||
'script': cls.redeem_script(signatures, pubkeys)
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def redeem_script(cls, signatures, pubkeys):
|
||||
return cls(template=cls.REDEEM_SCRIPT, values={
|
||||
'signatures_count': len(signatures),
|
||||
'pubkeys': pubkeys,
|
||||
'pubkeys_count': len(pubkeys)
|
||||
})
|
||||
|
||||
|
||||
class OutputScript(Script):
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
# output / payment script templates (aka scriptPubKey)
|
||||
PAY_PUBKEY_FULL = Template('pay_pubkey_full', (
|
||||
PUSH_SINGLE('pubkey'), OP_CHECKSIG
|
||||
))
|
||||
PAY_PUBKEY_HASH = Template('pay_pubkey_hash', (
|
||||
OP_DUP, OP_HASH160, PUSH_SINGLE('pubkey_hash'), OP_EQUALVERIFY, OP_CHECKSIG
|
||||
))
|
||||
PAY_SCRIPT_HASH = Template('pay_script_hash', (
|
||||
OP_HASH160, PUSH_SINGLE('script_hash'), OP_EQUAL
|
||||
))
|
||||
PAY_SEGWIT = Template('pay_script_hash+segwit', (
|
||||
OP_0, PUSH_SINGLE('script_hash')
|
||||
))
|
||||
RETURN_DATA = Template('return_data', (
|
||||
OP_RETURN, PUSH_SINGLE('data')
|
||||
))
|
||||
|
||||
CLAIM_NAME_OPCODES = (
|
||||
OP_CLAIM_NAME, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim'),
|
||||
OP_2DROP, OP_DROP
|
||||
)
|
||||
CLAIM_NAME_PUBKEY = Template('claim_name+pay_pubkey_hash', (
|
||||
CLAIM_NAME_OPCODES + PAY_PUBKEY_HASH.opcodes
|
||||
))
|
||||
CLAIM_NAME_SCRIPT = Template('claim_name+pay_script_hash', (
|
||||
CLAIM_NAME_OPCODES + PAY_SCRIPT_HASH.opcodes
|
||||
))
|
||||
|
||||
SUPPORT_CLAIM_OPCODES = (
|
||||
OP_SUPPORT_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'),
|
||||
OP_2DROP, OP_DROP
|
||||
)
|
||||
SUPPORT_CLAIM_PUBKEY = Template('support_claim+pay_pubkey_hash', (
|
||||
SUPPORT_CLAIM_OPCODES + PAY_PUBKEY_HASH.opcodes
|
||||
))
|
||||
SUPPORT_CLAIM_SCRIPT = Template('support_claim+pay_script_hash', (
|
||||
SUPPORT_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes
|
||||
))
|
||||
|
||||
SUPPORT_CLAIM_DATA_OPCODES = (
|
||||
OP_SUPPORT_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('support'),
|
||||
OP_2DROP, OP_2DROP
|
||||
)
|
||||
SUPPORT_CLAIM_DATA_PUBKEY = Template('support_claim+data+pay_pubkey_hash', (
|
||||
SUPPORT_CLAIM_DATA_OPCODES + PAY_PUBKEY_HASH.opcodes
|
||||
))
|
||||
SUPPORT_CLAIM_DATA_SCRIPT = Template('support_claim+data+pay_script_hash', (
|
||||
SUPPORT_CLAIM_DATA_OPCODES + PAY_SCRIPT_HASH.opcodes
|
||||
))
|
||||
|
||||
UPDATE_CLAIM_OPCODES = (
|
||||
OP_UPDATE_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim'),
|
||||
OP_2DROP, OP_2DROP
|
||||
)
|
||||
UPDATE_CLAIM_PUBKEY = Template('update_claim+pay_pubkey_hash', (
|
||||
UPDATE_CLAIM_OPCODES + PAY_PUBKEY_HASH.opcodes
|
||||
))
|
||||
UPDATE_CLAIM_SCRIPT = Template('update_claim+pay_script_hash', (
|
||||
UPDATE_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes
|
||||
))
|
||||
|
||||
SELL_SCRIPT = Template('sell_script', (
|
||||
OP_VERIFY, OP_DROP, OP_DROP, OP_DROP, PUSH_INTEGER('price'), OP_PRICECHECK
|
||||
))
|
||||
SELL_CLAIM = Template('sell_claim+pay_script_hash', (
|
||||
OP_SELL_CLAIM, PUSH_SINGLE('claim_id'), PUSH_SUBSCRIPT('sell_script', SELL_SCRIPT),
|
||||
PUSH_SUBSCRIPT('receive_script', InputScript.REDEEM_SCRIPT), OP_2DROP, OP_2DROP
|
||||
) + PAY_SCRIPT_HASH.opcodes)
|
||||
|
||||
BUY_CLAIM = Template('buy_claim+pay_script_hash', (
|
||||
OP_BUY_CLAIM, PUSH_SINGLE('sell_id'),
|
||||
PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim_version'),
|
||||
PUSH_SINGLE('owner_pubkey_hash'), PUSH_SINGLE('negotiation_signature'),
|
||||
OP_2DROP, OP_2DROP, OP_2DROP,
|
||||
) + PAY_SCRIPT_HASH.opcodes)
|
||||
|
||||
templates = [
|
||||
PAY_PUBKEY_FULL,
|
||||
PAY_PUBKEY_HASH,
|
||||
PAY_SCRIPT_HASH,
|
||||
PAY_SEGWIT,
|
||||
RETURN_DATA,
|
||||
CLAIM_NAME_PUBKEY,
|
||||
CLAIM_NAME_SCRIPT,
|
||||
SUPPORT_CLAIM_PUBKEY,
|
||||
SUPPORT_CLAIM_SCRIPT,
|
||||
SUPPORT_CLAIM_DATA_PUBKEY,
|
||||
SUPPORT_CLAIM_DATA_SCRIPT,
|
||||
UPDATE_CLAIM_PUBKEY,
|
||||
UPDATE_CLAIM_SCRIPT,
|
||||
SELL_CLAIM, SELL_SCRIPT,
|
||||
BUY_CLAIM,
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def pay_pubkey_hash(cls, pubkey_hash):
|
||||
return cls(template=cls.PAY_PUBKEY_HASH, values={
|
||||
'pubkey_hash': pubkey_hash
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def pay_script_hash(cls, script_hash):
|
||||
return cls(template=cls.PAY_SCRIPT_HASH, values={
|
||||
'script_hash': script_hash
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def return_data(cls, data):
|
||||
return cls(template=cls.RETURN_DATA, values={
|
||||
'data': data
|
||||
})
|
||||
|
||||
@property
|
||||
def is_pay_pubkey(self):
|
||||
return self.template.name.endswith('pay_pubkey_full')
|
||||
|
||||
@classmethod
|
||||
def pay_claim_name_pubkey_hash(cls, claim_name, claim, pubkey_hash):
|
||||
return cls(template=cls.CLAIM_NAME_PUBKEY, values={
|
||||
'claim_name': claim_name,
|
||||
'claim': claim,
|
||||
'pubkey_hash': pubkey_hash
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def pay_update_claim_pubkey_hash(cls, claim_name, claim_id, claim, pubkey_hash):
|
||||
return cls(template=cls.UPDATE_CLAIM_PUBKEY, values={
|
||||
'claim_name': claim_name,
|
||||
'claim_id': claim_id,
|
||||
'claim': claim,
|
||||
'pubkey_hash': pubkey_hash
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def pay_support_pubkey_hash(cls, claim_name: bytes, claim_id: bytes, pubkey_hash: bytes):
|
||||
return cls(template=cls.SUPPORT_CLAIM_PUBKEY, values={
|
||||
'claim_name': claim_name,
|
||||
'claim_id': claim_id,
|
||||
'pubkey_hash': pubkey_hash
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def pay_support_data_pubkey_hash(
|
||||
cls, claim_name: bytes, claim_id: bytes, support, pubkey_hash: bytes):
|
||||
return cls(template=cls.SUPPORT_CLAIM_DATA_PUBKEY, values={
|
||||
'claim_name': claim_name,
|
||||
'claim_id': claim_id,
|
||||
'support': support,
|
||||
'pubkey_hash': pubkey_hash
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def sell_script(cls, price):
|
||||
return cls(template=cls.SELL_SCRIPT, values={
|
||||
'price': price,
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def sell_claim(cls, claim_id, price, signatures, pubkeys):
|
||||
return cls(template=cls.SELL_CLAIM, values={
|
||||
'claim_id': claim_id,
|
||||
'sell_script': OutputScript.sell_script(price),
|
||||
'receive_script': InputScript.redeem_script(signatures, pubkeys)
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def buy_claim(cls, sell_id, claim_id, claim_version, owner_pubkey_hash, negotiation_signature):
|
||||
return cls(template=cls.BUY_CLAIM, values={
|
||||
'sell_id': sell_id,
|
||||
'claim_id': claim_id,
|
||||
'claim_version': claim_version,
|
||||
'owner_pubkey_hash': owner_pubkey_hash,
|
||||
'negotiation_signature': negotiation_signature,
|
||||
})
|
||||
|
||||
@property
|
||||
def is_pay_pubkey_hash(self):
|
||||
return self.template.name.endswith('pay_pubkey_hash')
|
||||
|
||||
@property
|
||||
def is_pay_script_hash(self):
|
||||
return self.template.name.endswith('pay_script_hash')
|
||||
|
||||
@property
|
||||
def is_return_data(self):
|
||||
return self.template.name.endswith('return_data')
|
||||
|
||||
@property
|
||||
def is_claim_name(self):
|
||||
return self.template.name.startswith('claim_name+')
|
||||
|
||||
@property
|
||||
def is_update_claim(self):
|
||||
return self.template.name.startswith('update_claim+')
|
||||
|
||||
@property
|
||||
def is_support_claim(self):
|
||||
return self.template.name.startswith('support_claim+')
|
||||
|
||||
@property
|
||||
def is_support_claim_data(self):
|
||||
return self.template.name.startswith('support_claim+data+')
|
||||
|
||||
@property
|
||||
def is_sell_claim(self):
|
||||
return self.template.name.startswith('sell_claim+')
|
||||
|
||||
@property
|
||||
def is_buy_claim(self):
|
||||
return self.template.name.startswith('buy_claim+')
|
||||
|
||||
@property
|
||||
def is_claim_involved(self):
|
||||
return any((
|
||||
self.is_claim_name, self.is_support_claim, self.is_update_claim,
|
||||
self.is_sell_claim, self.is_buy_claim
|
||||
))
|
||||
@@ -1 +0,0 @@
|
||||
from .synchronizer import BlockchainSync
|
||||
@@ -1,340 +0,0 @@
|
||||
import logging
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import Tuple, List
|
||||
|
||||
from sqlalchemy import table, text, func, union, between
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy.schema import CreateTable
|
||||
|
||||
from lbry.db.tables import (
|
||||
Block as BlockTable, BlockFilter,
|
||||
TX, TXFilter, TXO, TXI, Claim, Tag, Support
|
||||
)
|
||||
from lbry.db.tables import (
|
||||
pg_add_block_constraints_and_indexes,
|
||||
pg_add_block_filter_constraints_and_indexes,
|
||||
pg_add_tx_constraints_and_indexes,
|
||||
pg_add_tx_filter_constraints_and_indexes,
|
||||
pg_add_txo_constraints_and_indexes,
|
||||
pg_add_txi_constraints_and_indexes,
|
||||
)
|
||||
from lbry.db.query_context import ProgressContext, event_emitter, context
|
||||
from lbry.db.sync import set_input_addresses, update_spent_outputs
|
||||
from lbry.blockchain.transaction import Transaction
|
||||
from lbry.blockchain.block import Block, create_address_filter
|
||||
from lbry.blockchain.bcd_data_stream import BCDataStream
|
||||
|
||||
from .context import get_or_initialize_lbrycrd
|
||||
from .filter_builder import FilterBuilder
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_best_block_height_for_file(file_number):
|
||||
return context().fetchone(
|
||||
select(func.coalesce(func.max(BlockTable.c.height), -1).label('height'))
|
||||
.where(BlockTable.c.file_number == file_number)
|
||||
)['height']
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.blocks.file", "blocks", "txs", throttle=100)
|
||||
def sync_block_file(
|
||||
file_number: int, start_height: int, txs: int, flush_size: int, p: ProgressContext
|
||||
):
|
||||
chain = get_or_initialize_lbrycrd(p.ctx)
|
||||
new_blocks = chain.db.sync_get_blocks_in_file(file_number, start_height)
|
||||
if not new_blocks:
|
||||
return -1
|
||||
file_name = chain.get_block_file_name(file_number)
|
||||
p.start(len(new_blocks), txs, progress_id=file_number, label=file_name)
|
||||
block_file_path = chain.get_block_file_path(file_number)
|
||||
done_blocks = done_txs = 0
|
||||
last_block_processed, loader = -1, p.ctx.get_bulk_loader()
|
||||
with open(block_file_path, "rb") as fp:
|
||||
stream = BCDataStream(fp=fp)
|
||||
for done_blocks, block_info in enumerate(new_blocks, start=1):
|
||||
block_height = block_info["height"]
|
||||
fp.seek(block_info["data_offset"])
|
||||
block = Block.from_data_stream(stream, block_height, file_number)
|
||||
loader.add_block(block)
|
||||
if len(loader.txs) >= flush_size:
|
||||
done_txs += loader.flush(TX)
|
||||
p.step(done_blocks, done_txs)
|
||||
last_block_processed = block_height
|
||||
if p.ctx.stop_event.is_set():
|
||||
return last_block_processed
|
||||
if loader.txs:
|
||||
done_txs += loader.flush(TX)
|
||||
p.step(done_blocks, done_txs)
|
||||
return last_block_processed
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.blocks.indexes", "steps")
|
||||
def blocks_constraints_and_indexes(p: ProgressContext):
|
||||
p.start(1 + len(pg_add_block_constraints_and_indexes))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE block;"))
|
||||
p.step()
|
||||
for constraint in pg_add_block_constraints_and_indexes:
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.blocks.vacuum", "steps")
|
||||
def blocks_vacuum(p: ProgressContext):
|
||||
p.start(1)
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM block;"))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.spends.main", "steps")
|
||||
def sync_spends(initial_sync: bool, p: ProgressContext):
|
||||
if initial_sync:
|
||||
p.start(
|
||||
7 +
|
||||
len(pg_add_tx_constraints_and_indexes) +
|
||||
len(pg_add_txi_constraints_and_indexes) +
|
||||
len(pg_add_txo_constraints_and_indexes)
|
||||
)
|
||||
# 1. tx table stuff
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE tx;"))
|
||||
p.step()
|
||||
for constraint in pg_add_tx_constraints_and_indexes:
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
# A. Update TXIs to have the address of TXO they are spending.
|
||||
# 2. txi table reshuffling
|
||||
p.ctx.execute(text("ALTER TABLE txi RENAME TO old_txi;"))
|
||||
p.ctx.execute(CreateTable(TXI, include_foreign_key_constraints=[]))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text("ALTER TABLE txi DROP CONSTRAINT txi_pkey;"))
|
||||
p.step()
|
||||
# 3. insert
|
||||
old_txi = table("old_txi", *(c.copy() for c in TXI.columns)) # pylint: disable=not-an-iterable
|
||||
columns = [c for c in old_txi.columns if c.name != "address"] + [TXO.c.address]
|
||||
join_txi_on_txo = old_txi.join(TXO, old_txi.c.txo_hash == TXO.c.txo_hash)
|
||||
select_txis = select(*columns).select_from(join_txi_on_txo)
|
||||
insert_txis = TXI.insert().from_select(columns, select_txis)
|
||||
p.ctx.execute(insert_txis)
|
||||
p.step()
|
||||
# 4. drop old txi and vacuum
|
||||
p.ctx.execute(text("DROP TABLE old_txi;"))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE txi;"))
|
||||
p.step()
|
||||
for constraint in pg_add_txi_constraints_and_indexes:
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
# B. Update TXOs to have the height at which they were spent (if they were).
|
||||
# 5. txo table reshuffling
|
||||
p.ctx.execute(text("ALTER TABLE txo RENAME TO old_txo;"))
|
||||
p.ctx.execute(CreateTable(TXO, include_foreign_key_constraints=[]))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text("ALTER TABLE txo DROP CONSTRAINT txo_pkey;"))
|
||||
p.step()
|
||||
# 6. insert
|
||||
old_txo = table("old_txo", *(c.copy() for c in TXO.columns)) # pylint: disable=not-an-iterable
|
||||
columns = [c for c in old_txo.columns if c.name != "spent_height"]
|
||||
insert_columns = columns + [TXO.c.spent_height]
|
||||
select_columns = columns + [func.coalesce(TXI.c.height, 0).label("spent_height")]
|
||||
join_txo_on_txi = old_txo.join(TXI, old_txo.c.txo_hash == TXI.c.txo_hash, isouter=True)
|
||||
select_txos = select(*select_columns).select_from(join_txo_on_txi)
|
||||
insert_txos = TXO.insert().from_select(insert_columns, select_txos)
|
||||
p.ctx.execute(insert_txos)
|
||||
p.step()
|
||||
# 7. drop old txo
|
||||
p.ctx.execute(text("DROP TABLE old_txo;"))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE txo;"))
|
||||
p.step()
|
||||
for constraint in pg_add_txo_constraints_and_indexes:
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
else:
|
||||
p.start(5)
|
||||
# 1. Update spent TXOs setting spent_height
|
||||
update_spent_outputs(p.ctx)
|
||||
p.step()
|
||||
# 2. Update TXIs to have the address of TXO they are spending.
|
||||
set_input_addresses(p.ctx)
|
||||
p.step()
|
||||
# 3. Update tx visibility map, which speeds up index-only scans.
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM tx;"))
|
||||
p.step()
|
||||
# 4. Update txi visibility map, which speeds up index-only scans.
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM txi;"))
|
||||
p.step()
|
||||
# 4. Update txo visibility map, which speeds up index-only scans.
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM txo;"))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.mempool.clear", "txs")
|
||||
def clear_mempool(p: ProgressContext):
|
||||
delete_all_the_things(-1, p)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.mempool.main", "txs")
|
||||
def sync_mempool(p: ProgressContext) -> List[str]:
|
||||
chain = get_or_initialize_lbrycrd(p.ctx)
|
||||
mempool = chain.sync_run(chain.get_raw_mempool())
|
||||
current = [hexlify(r['tx_hash'][::-1]).decode() for r in p.ctx.fetchall(
|
||||
select(TX.c.tx_hash).where(TX.c.height < 0)
|
||||
)]
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
added = []
|
||||
for txid in mempool:
|
||||
if txid not in current:
|
||||
raw_tx = chain.sync_run(chain.get_raw_transaction(txid))
|
||||
loader.add_transaction(
|
||||
None, Transaction(unhexlify(raw_tx), height=-1)
|
||||
)
|
||||
added.append(txid)
|
||||
if p.ctx.stop_event.is_set():
|
||||
return
|
||||
loader.flush(TX)
|
||||
return added
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.filters.generate", "blocks", throttle=100)
|
||||
def sync_filters(start, end, p: ProgressContext):
|
||||
fp = FilterBuilder(start, end)
|
||||
p.start((end-start)+1, progress_id=start, label=f"generate filters {start}-{end}")
|
||||
with p.ctx.connect_streaming() as c:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
|
||||
tx_hash, height, addresses, last_added = None, None, set(), None
|
||||
address_to_hash = p.ctx.ledger.address_to_hash160
|
||||
for row in c.execute(get_block_tx_addresses_sql(*fp.query_heights)):
|
||||
if tx_hash != row.tx_hash:
|
||||
if tx_hash is not None:
|
||||
last_added = tx_hash
|
||||
fp.add(tx_hash, height, addresses)
|
||||
tx_hash, height, addresses = row.tx_hash, row.height, set()
|
||||
addresses.add(address_to_hash(row.address))
|
||||
if all([last_added, tx_hash]) and last_added != tx_hash: # pickup last tx
|
||||
fp.add(tx_hash, height, addresses)
|
||||
|
||||
for tx_hash, height, addresses in fp.tx_filters:
|
||||
loader.add_transaction_filter(
|
||||
tx_hash, height, create_address_filter(list(addresses))
|
||||
)
|
||||
|
||||
for height, addresses in fp.block_filters.items():
|
||||
loader.add_block_filter(
|
||||
height, 1, create_address_filter(list(addresses))
|
||||
)
|
||||
|
||||
for group_filter in fp.group_filters:
|
||||
for height, addresses in group_filter.groups.items():
|
||||
loader.add_block_filter(
|
||||
height, group_filter.factor, create_address_filter(list(addresses))
|
||||
)
|
||||
|
||||
p.add(loader.flush(BlockFilter))
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.filters.indexes", "steps")
|
||||
def filters_constraints_and_indexes(p: ProgressContext):
|
||||
is_postgres = p.ctx.is_postgres and p.ctx.pg_has_pk_constraint('block_filter')
|
||||
constraints = (
|
||||
pg_add_tx_filter_constraints_and_indexes +
|
||||
pg_add_block_filter_constraints_and_indexes
|
||||
)
|
||||
p.start(2 + len(constraints))
|
||||
if is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE block_filter;"))
|
||||
p.step()
|
||||
if is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE tx_filter;"))
|
||||
p.step()
|
||||
for constraint in constraints:
|
||||
if is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.filters.vacuum", "steps")
|
||||
def filters_vacuum(p: ProgressContext):
|
||||
p.start(2)
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM block_filter;"))
|
||||
p.step()
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM tx_filter;"))
|
||||
p.step()
|
||||
|
||||
|
||||
def get_block_range_without_filters() -> Tuple[int, int]:
|
||||
sql = (
|
||||
select(
|
||||
func.coalesce(func.min(BlockTable.c.height), -1).label('start_height'),
|
||||
func.coalesce(func.max(BlockTable.c.height), -1).label('end_height'),
|
||||
)
|
||||
.select_from(
|
||||
BlockTable.join(
|
||||
BlockFilter,
|
||||
(BlockTable.c.height == BlockFilter.c.height) &
|
||||
(BlockFilter.c.factor == 1),
|
||||
isouter=True
|
||||
)
|
||||
)
|
||||
.where(BlockFilter.c.height.is_(None))
|
||||
)
|
||||
result = context().fetchone(sql)
|
||||
return result['start_height'], result['end_height']
|
||||
|
||||
|
||||
def get_block_tx_addresses_sql(start_height, end_height):
|
||||
return union(
|
||||
select(TXO.c.tx_hash, TXO.c.height, TXO.c.address).where(
|
||||
(TXO.c.address.isnot(None)) & between(TXO.c.height, start_height, end_height)
|
||||
),
|
||||
select(TXI.c.tx_hash, TXI.c.height, TXI.c.address).where(
|
||||
(TXI.c.address.isnot(None)) & between(TXI.c.height, start_height, end_height)
|
||||
),
|
||||
).order_by('height', 'tx_hash')
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.rewind.main", "steps")
|
||||
def rewind(height: int, p: ProgressContext):
|
||||
delete_all_the_things(height, p)
|
||||
|
||||
|
||||
def delete_all_the_things(height: int, p: ProgressContext):
|
||||
def constrain(col):
|
||||
if height == -1:
|
||||
return col == -1
|
||||
return col >= height
|
||||
|
||||
deletes = [
|
||||
BlockTable.delete().where(constrain(BlockTable.c.height)),
|
||||
TXI.delete().where(constrain(TXI.c.height)),
|
||||
TXO.delete().where(constrain(TXO.c.height)),
|
||||
TX.delete().where(constrain(TX.c.height)),
|
||||
Tag.delete().where(
|
||||
Tag.c.claim_hash.in_(
|
||||
select(Claim.c.claim_hash).where(constrain(Claim.c.height))
|
||||
)
|
||||
),
|
||||
Claim.delete().where(constrain(Claim.c.height)),
|
||||
Support.delete().where(constrain(Support.c.height)),
|
||||
]
|
||||
if height > 0:
|
||||
deletes.extend([
|
||||
# TODO: block and tx filters need better where() clauses (below actually breaks things)
|
||||
BlockFilter.delete().where(BlockFilter.c.height >= height),
|
||||
TXFilter.delete(),
|
||||
])
|
||||
for delete in p.iter(deletes):
|
||||
p.ctx.execute(delete)
|
||||
@@ -1,335 +0,0 @@
|
||||
import logging
|
||||
from typing import Tuple
|
||||
|
||||
from sqlalchemy import case, func, text
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from lbry.db.queries.txio import (
|
||||
minimum_txo_columns, row_to_txo,
|
||||
where_unspent_txos, where_claims_with_changed_supports,
|
||||
count_unspent_txos, where_channels_with_changed_content,
|
||||
where_abandoned_claims, count_channels_with_changed_content,
|
||||
where_claims_with_changed_reposts, where_claims_with_stale_signatures
|
||||
)
|
||||
from lbry.db.query_context import ProgressContext, event_emitter
|
||||
from lbry.db.tables import (
|
||||
TX, TXO, Claim, Support, CensoredClaim,
|
||||
pg_add_claim_and_tag_constraints_and_indexes
|
||||
)
|
||||
from lbry.db.utils import least
|
||||
from lbry.db.constants import TXO_TYPES, CLAIM_TYPE_CODES
|
||||
from lbry.schema.result import Censor
|
||||
from lbry.blockchain.transaction import Output
|
||||
|
||||
from .context import get_or_initialize_lbrycrd
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def channel_content_count_calc(signable):
|
||||
return (
|
||||
select(func.count(signable.c.claim_hash))
|
||||
.where((signable.c.channel_hash == Claim.c.claim_hash) & signable.c.is_signature_valid)
|
||||
.scalar_subquery()
|
||||
)
|
||||
|
||||
|
||||
support = TXO.alias('support')
|
||||
|
||||
|
||||
def staked_support_aggregation(aggregate):
|
||||
return (
|
||||
select(aggregate).where(
|
||||
(support.c.txo_type == TXO_TYPES['support']) &
|
||||
(support.c.spent_height == 0)
|
||||
).scalar_subquery()
|
||||
)
|
||||
|
||||
|
||||
def staked_support_amount_calc(other):
|
||||
return (
|
||||
staked_support_aggregation(func.coalesce(func.sum(support.c.amount), 0))
|
||||
.where(support.c.claim_hash == other.c.claim_hash)
|
||||
)
|
||||
|
||||
|
||||
def staked_support_count_calc(other):
|
||||
return (
|
||||
staked_support_aggregation(func.coalesce(func.count('*'), 0))
|
||||
.where(support.c.claim_hash == other.c.claim_hash)
|
||||
)
|
||||
|
||||
|
||||
def reposted_claim_count_calc(other):
|
||||
repost = TXO.alias('repost')
|
||||
return (
|
||||
select(func.coalesce(func.count(repost.c.reposted_claim_hash), 0))
|
||||
.where(
|
||||
(repost.c.reposted_claim_hash == other.c.claim_hash) &
|
||||
(repost.c.spent_height == 0)
|
||||
).scalar_subquery()
|
||||
)
|
||||
|
||||
|
||||
def make_label(action, blocks):
|
||||
if blocks[0] == blocks[-1]:
|
||||
return f"{action} {blocks[0]:>6}"
|
||||
else:
|
||||
return f"{action} {blocks[0]:>6}-{blocks[-1]:>6}"
|
||||
|
||||
|
||||
def select_claims_for_saving(
|
||||
blocks: Tuple[int, int],
|
||||
missing_in_claims_table=False,
|
||||
missing_or_stale_in_claims_table=False,
|
||||
):
|
||||
channel_txo = TXO.alias('channel_txo')
|
||||
return select(
|
||||
*minimum_txo_columns, TXO.c.claim_hash,
|
||||
staked_support_amount_calc(TXO).label('staked_support_amount'),
|
||||
staked_support_count_calc(TXO).label('staked_support_count'),
|
||||
reposted_claim_count_calc(TXO).label('reposted_count'),
|
||||
TXO.c.signature, TXO.c.signature_digest,
|
||||
case([(
|
||||
TXO.c.channel_hash.isnot(None),
|
||||
select(channel_txo.c.public_key).select_from(channel_txo).where(
|
||||
(channel_txo.c.spent_height == 0) &
|
||||
(channel_txo.c.txo_type == TXO_TYPES['channel']) &
|
||||
(channel_txo.c.claim_hash == TXO.c.channel_hash)
|
||||
).limit(1).scalar_subquery()
|
||||
)]).label('channel_public_key')
|
||||
).where(
|
||||
where_unspent_txos(
|
||||
CLAIM_TYPE_CODES, blocks,
|
||||
missing_in_claims_table=missing_in_claims_table,
|
||||
missing_or_stale_in_claims_table=missing_or_stale_in_claims_table,
|
||||
)
|
||||
).select_from(TXO.join(TX))
|
||||
|
||||
|
||||
def row_to_claim_for_saving(row) -> Tuple[Output, dict]:
|
||||
return row_to_txo(row), {
|
||||
'staked_support_amount': int(row.staked_support_amount),
|
||||
'staked_support_count': int(row.staked_support_count),
|
||||
'reposted_count': int(row.reposted_count),
|
||||
'signature': row.signature,
|
||||
'signature_digest': row.signature_digest,
|
||||
'channel_public_key': row.channel_public_key
|
||||
}
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.insert", "claims")
|
||||
def claims_insert(
|
||||
blocks: Tuple[int, int],
|
||||
missing_in_claims_table: bool,
|
||||
flush_size: int,
|
||||
p: ProgressContext
|
||||
):
|
||||
chain = get_or_initialize_lbrycrd(p.ctx)
|
||||
|
||||
p.start(
|
||||
count_unspent_txos(
|
||||
CLAIM_TYPE_CODES, blocks,
|
||||
missing_in_claims_table=missing_in_claims_table,
|
||||
), progress_id=blocks[0], label=make_label("add claims", blocks)
|
||||
)
|
||||
|
||||
with p.ctx.connect_streaming() as c:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
cursor = c.execute(select_claims_for_saving(
|
||||
blocks, missing_in_claims_table=missing_in_claims_table
|
||||
).order_by(TXO.c.claim_hash))
|
||||
for rows in cursor.partitions(900):
|
||||
claim_metadata = chain.db.sync_get_claim_metadata(
|
||||
claim_hashes=[row['claim_hash'] for row in rows]
|
||||
)
|
||||
i = 0
|
||||
for row in rows:
|
||||
metadata = claim_metadata[i] if i < len(claim_metadata) else {}
|
||||
if metadata and metadata['claim_hash'] == row.claim_hash:
|
||||
i += 1
|
||||
txo, extra = row_to_claim_for_saving(row)
|
||||
extra.update({
|
||||
'short_url': metadata.get('short_url'),
|
||||
'creation_height': metadata.get('creation_height'),
|
||||
'activation_height': metadata.get('activation_height'),
|
||||
'expiration_height': metadata.get('expiration_height'),
|
||||
'takeover_height': metadata.get('takeover_height'),
|
||||
})
|
||||
loader.add_claim(txo, **extra)
|
||||
if len(loader.claims) >= flush_size:
|
||||
p.add(loader.flush(Claim))
|
||||
p.add(loader.flush(Claim))
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.indexes", "steps")
|
||||
def claims_constraints_and_indexes(p: ProgressContext):
|
||||
should_run = p.ctx.is_postgres and not p.ctx.pg_has_pk_constraint('claim')
|
||||
p.start(2 + len(pg_add_claim_and_tag_constraints_and_indexes))
|
||||
if should_run:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE claim;"))
|
||||
p.step()
|
||||
if should_run:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE tag;"))
|
||||
p.step()
|
||||
for constraint in pg_add_claim_and_tag_constraints_and_indexes:
|
||||
if should_run:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.vacuum", "steps")
|
||||
def claims_vacuum(p: ProgressContext):
|
||||
p.start(2)
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM claim;"))
|
||||
p.step()
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM tag;"))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.update", "claims")
|
||||
def claims_update(blocks: Tuple[int, int], p: ProgressContext):
|
||||
p.start(
|
||||
count_unspent_txos(CLAIM_TYPE_CODES, blocks, missing_or_stale_in_claims_table=True),
|
||||
progress_id=blocks[0], label=make_label("mod claims", blocks)
|
||||
)
|
||||
with p.ctx.connect_streaming() as c:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
cursor = c.execute(select_claims_for_saving(
|
||||
blocks, missing_or_stale_in_claims_table=True
|
||||
))
|
||||
for row in cursor:
|
||||
txo, extra = row_to_claim_for_saving(row)
|
||||
loader.update_claim(txo, **extra)
|
||||
if len(loader.update_claims) >= 25:
|
||||
p.add(loader.flush(Claim))
|
||||
p.add(loader.flush(Claim))
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.delete", "claims")
|
||||
def claims_delete(claims, p: ProgressContext):
|
||||
p.start(claims, label="del claims")
|
||||
deleted = p.ctx.execute(Claim.delete().where(where_abandoned_claims()))
|
||||
p.step(deleted.rowcount)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.takeovers", "claims")
|
||||
def update_takeovers(blocks: Tuple[int, int], takeovers, p: ProgressContext):
|
||||
p.start(takeovers, label=make_label("mod winner", blocks))
|
||||
chain = get_or_initialize_lbrycrd(p.ctx)
|
||||
with p.ctx.engine.begin() as c:
|
||||
for takeover in chain.db.sync_get_takeovers(start_height=blocks[0], end_height=blocks[-1]):
|
||||
update_claims = (
|
||||
Claim.update()
|
||||
.where(Claim.c.normalized == takeover['normalized'])
|
||||
.values(
|
||||
is_controlling=case(
|
||||
[(Claim.c.claim_hash == takeover['claim_hash'], True)],
|
||||
else_=False
|
||||
),
|
||||
takeover_height=case(
|
||||
[(Claim.c.claim_hash == takeover['claim_hash'], takeover['height'])],
|
||||
else_=None
|
||||
),
|
||||
activation_height=least(Claim.c.activation_height, takeover['height']),
|
||||
)
|
||||
)
|
||||
result = c.execute(update_claims)
|
||||
p.add(result.rowcount)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.stakes", "claims")
|
||||
def update_stakes(blocks: Tuple[int, int], claims: int, p: ProgressContext):
|
||||
p.start(claims)
|
||||
sql = (
|
||||
Claim.update()
|
||||
.where(where_claims_with_changed_supports(blocks))
|
||||
.values(
|
||||
staked_support_amount=staked_support_amount_calc(Claim),
|
||||
staked_support_count=staked_support_count_calc(Claim),
|
||||
)
|
||||
)
|
||||
result = p.ctx.execute(sql)
|
||||
p.step(result.rowcount)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.reposts", "claims")
|
||||
def update_reposts(blocks: Tuple[int, int], claims: int, p: ProgressContext):
|
||||
p.start(claims)
|
||||
sql = (
|
||||
Claim.update()
|
||||
.where(where_claims_with_changed_reposts(blocks))
|
||||
.values(reposted_count=reposted_claim_count_calc(Claim))
|
||||
)
|
||||
result = p.ctx.execute(sql)
|
||||
p.step(result.rowcount)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.invalidate", "claims")
|
||||
def update_stale_signatures(blocks: Tuple[int, int], claims: int, p: ProgressContext):
|
||||
p.start(claims)
|
||||
with p.ctx.connect_streaming() as c:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
stream = Claim.alias('stream')
|
||||
sql = (
|
||||
select_claims_for_saving(None)
|
||||
.where(TXO.c.claim_hash.in_(
|
||||
where_claims_with_stale_signatures(
|
||||
select(stream.c.claim_hash), blocks, stream
|
||||
)
|
||||
))
|
||||
)
|
||||
cursor = c.execute(sql)
|
||||
for row in cursor:
|
||||
txo, extra = row_to_claim_for_saving(row)
|
||||
loader.update_claim(txo, public_key_height=blocks[1], **extra)
|
||||
if len(loader.update_claims) >= 25:
|
||||
p.add(loader.flush(Claim))
|
||||
p.add(loader.flush(Claim))
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.channels", "channels")
|
||||
def update_channel_stats(blocks: Tuple[int, int], initial_sync: int, p: ProgressContext):
|
||||
update_sql = Claim.update().values(
|
||||
signed_claim_count=channel_content_count_calc(Claim.alias('content')),
|
||||
signed_support_count=channel_content_count_calc(Support),
|
||||
)
|
||||
if initial_sync:
|
||||
p.start(p.ctx.fetchtotal(Claim.c.claim_type == TXO_TYPES['channel']), label="channel stats")
|
||||
update_sql = update_sql.where(Claim.c.claim_type == TXO_TYPES['channel'])
|
||||
elif blocks:
|
||||
p.start(count_channels_with_changed_content(blocks), label="channel stats")
|
||||
update_sql = update_sql.where(where_channels_with_changed_content(blocks))
|
||||
else:
|
||||
return
|
||||
result = p.ctx.execute(update_sql)
|
||||
if result.rowcount and p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM claim;"))
|
||||
p.step(result.rowcount)
|
||||
|
||||
|
||||
def select_reposts(channel_hashes, filter_type):
|
||||
return (
|
||||
select(Claim.c.reposted_claim_hash, filter_type, Claim.c.channel_hash).where(
|
||||
(Claim.c.channel_hash.in_(channel_hashes)) &
|
||||
(Claim.c.reposted_claim_hash.isnot(None))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.claims.filters", "claim_filters")
|
||||
def update_claim_filters(resolve_censor_channel_hashes, search_censor_channel_hashes, p: ProgressContext):
|
||||
p.ctx.execute(CensoredClaim.delete())
|
||||
# order matters: first we insert the resolve filters; then the search ones.
|
||||
# a claim that's censored in resolve is automatically also censored in search results.
|
||||
p.ctx.execute(CensoredClaim.insert().from_select(
|
||||
['claim_hash', 'censor_type', 'censoring_channel_hash'],
|
||||
select_reposts(resolve_censor_channel_hashes, Censor.RESOLVE)
|
||||
))
|
||||
p.ctx.execute(p.ctx.insert_or_ignore(CensoredClaim).from_select(
|
||||
['claim_hash', 'censor_type', 'censoring_channel_hash'],
|
||||
select_reposts(search_censor_channel_hashes, Censor.SEARCH)
|
||||
))
|
||||
@@ -1,25 +0,0 @@
|
||||
from contextvars import ContextVar
|
||||
from lbry.db import query_context
|
||||
|
||||
from lbry.blockchain.lbrycrd import Lbrycrd
|
||||
|
||||
|
||||
_chain: ContextVar[Lbrycrd] = ContextVar('chain')
|
||||
|
||||
|
||||
def get_or_initialize_lbrycrd(ctx=None) -> Lbrycrd:
|
||||
chain = _chain.get(None)
|
||||
if chain is not None:
|
||||
return chain
|
||||
chain = Lbrycrd((ctx or query_context.context()).ledger)
|
||||
chain.db.sync_open()
|
||||
_chain.set(chain)
|
||||
return chain
|
||||
|
||||
|
||||
def uninitialize():
|
||||
chain = _chain.get(None)
|
||||
if chain is not None:
|
||||
chain.db.sync_close()
|
||||
chain.sync_run(chain.close_session())
|
||||
_chain.set(None)
|
||||
@@ -1,80 +0,0 @@
|
||||
from typing import Dict
|
||||
|
||||
|
||||
def split_range_into_batches(start, end, batch_size=100_000):
|
||||
batch = [start, end]
|
||||
batches = [batch]
|
||||
for block in range(start, end+1):
|
||||
if 0 < block != batch[0] and block % batch_size == 0:
|
||||
batch = [block, block]
|
||||
batches.append(batch)
|
||||
else:
|
||||
batch[1] = block
|
||||
return batches
|
||||
|
||||
|
||||
class GroupFilter:
|
||||
"""
|
||||
Collects addresses into buckets of specific sizes defined by 10 raised to power of factor.
|
||||
eg. a factor of 2 (10**2) would create block buckets 100-199, 200-299, etc
|
||||
a factor of 3 (10**3) would create block buckets 1000-1999, 2000-2999, etc
|
||||
"""
|
||||
def __init__(self, start, end, factor):
|
||||
self.start = start
|
||||
self.end = end
|
||||
self.factor = factor
|
||||
self.resolution = resolution = 10**factor
|
||||
last_height_in_group, groups = resolution-1, {}
|
||||
for block in range(start, end+1):
|
||||
if block % resolution == last_height_in_group:
|
||||
groups[block-last_height_in_group] = set()
|
||||
self.last_height_in_group = last_height_in_group
|
||||
self.groups: Dict[int, set] = groups
|
||||
|
||||
@property
|
||||
def coverage(self):
|
||||
return list(self.groups.keys())
|
||||
|
||||
def add(self, height, addresses):
|
||||
group = self.groups.get(height - (height % self.resolution))
|
||||
if group is not None:
|
||||
group.update(addresses)
|
||||
|
||||
|
||||
class FilterBuilder:
|
||||
"""
|
||||
Creates filter groups, calculates the necessary block range to fulfill creation
|
||||
of filter groups and collects tx filters, block filters and group filters.
|
||||
"""
|
||||
def __init__(self, start, end):
|
||||
self.start = start
|
||||
self.end = end
|
||||
self.group_filters = [
|
||||
GroupFilter(start, end, 5),
|
||||
GroupFilter(start, end, 4),
|
||||
GroupFilter(start, end, 3),
|
||||
GroupFilter(start, end, 2),
|
||||
]
|
||||
self.start_tx_height, self.end_tx_height = self._calculate_tx_heights_for_query()
|
||||
self.tx_filters = []
|
||||
self.block_filters: Dict[int, set] = {}
|
||||
|
||||
def _calculate_tx_heights_for_query(self):
|
||||
for group_filter in self.group_filters:
|
||||
if group_filter.groups:
|
||||
return group_filter.coverage[0], self.end
|
||||
return self.start, self.end
|
||||
|
||||
@property
|
||||
def query_heights(self):
|
||||
return self.start_tx_height, self.end_tx_height
|
||||
|
||||
def add(self, tx_hash, height, addresses):
|
||||
if self.start <= height <= self.end:
|
||||
self.tx_filters.append((tx_hash, height, addresses))
|
||||
block_filter = self.block_filters.get(height)
|
||||
if block_filter is None:
|
||||
block_filter = self.block_filters[height] = set()
|
||||
block_filter.update(addresses)
|
||||
for group_filter in self.group_filters:
|
||||
group_filter.add(height, addresses)
|
||||
@@ -1,95 +0,0 @@
|
||||
import logging
|
||||
from typing import Tuple
|
||||
|
||||
from sqlalchemy import case, desc, text
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from lbry.db.tables import TX, TXO, Support, pg_add_support_constraints_and_indexes
|
||||
from lbry.db.query_context import ProgressContext, event_emitter
|
||||
from lbry.db.queries import row_to_txo
|
||||
from lbry.db.constants import TXO_TYPES
|
||||
from lbry.db.queries.txio import (
|
||||
minimum_txo_columns,
|
||||
where_unspent_txos, where_abandoned_supports,
|
||||
count_unspent_txos,
|
||||
)
|
||||
|
||||
from .claims import make_label
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.supports.insert", "supports")
|
||||
def supports_insert(
|
||||
blocks: Tuple[int, int],
|
||||
missing_in_supports_table: bool,
|
||||
flush_size: int,
|
||||
p: ProgressContext
|
||||
):
|
||||
p.start(
|
||||
count_unspent_txos(
|
||||
TXO_TYPES['support'], blocks,
|
||||
missing_in_supports_table=missing_in_supports_table,
|
||||
), progress_id=blocks[0], label=make_label("add supprt", blocks)
|
||||
)
|
||||
channel_txo = TXO.alias('channel_txo')
|
||||
select_supports = select(
|
||||
*minimum_txo_columns, TXO.c.claim_hash,
|
||||
TXO.c.signature, TXO.c.signature_digest,
|
||||
case([(
|
||||
TXO.c.channel_hash.isnot(None),
|
||||
select(channel_txo.c.public_key).select_from(channel_txo).where(
|
||||
(channel_txo.c.txo_type == TXO_TYPES['channel']) &
|
||||
(channel_txo.c.claim_hash == TXO.c.channel_hash) &
|
||||
(channel_txo.c.height <= TXO.c.height)
|
||||
).order_by(desc(channel_txo.c.height)).limit(1).scalar_subquery()
|
||||
)]).label('channel_public_key'),
|
||||
).select_from(
|
||||
TXO.join(TX)
|
||||
).where(
|
||||
where_unspent_txos(
|
||||
TXO_TYPES['support'], blocks,
|
||||
missing_in_supports_table=missing_in_supports_table,
|
||||
)
|
||||
)
|
||||
with p.ctx.connect_streaming() as c:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
for row in c.execute(select_supports):
|
||||
txo = row_to_txo(row)
|
||||
loader.add_support(
|
||||
txo,
|
||||
signature=row.signature,
|
||||
signature_digest=row.signature_digest,
|
||||
channel_public_key=row.channel_public_key
|
||||
)
|
||||
if len(loader.supports) >= flush_size:
|
||||
p.add(loader.flush(Support))
|
||||
p.add(loader.flush(Support))
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.supports.delete", "supports")
|
||||
def supports_delete(supports, p: ProgressContext):
|
||||
p.start(supports, label="del supprt")
|
||||
deleted = p.ctx.execute(Support.delete().where(where_abandoned_supports()))
|
||||
p.step(deleted.rowcount)
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.supports.indexes", "steps")
|
||||
def supports_constraints_and_indexes(p: ProgressContext):
|
||||
p.start(1 + len(pg_add_support_constraints_and_indexes))
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM ANALYZE support;"))
|
||||
p.step()
|
||||
for constraint in pg_add_support_constraints_and_indexes:
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute(text(constraint))
|
||||
p.step()
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.supports.vacuum", "steps")
|
||||
def supports_vacuum(p: ProgressContext):
|
||||
p.start(1)
|
||||
if p.ctx.is_postgres:
|
||||
p.ctx.execute_notx(text("VACUUM support;"))
|
||||
p.step()
|
||||
@@ -1,422 +0,0 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
from typing import Optional, Tuple, Set, List, Coroutine
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from lbry.db import Database, trending
|
||||
from lbry.db import queries as q
|
||||
from lbry.db.constants import TXO_TYPES, CLAIM_TYPE_CODES
|
||||
from lbry.db.query_context import Event, Progress
|
||||
from lbry.event import BroadcastSubscription, EventController
|
||||
from lbry.service.base import Sync, BlockEvent
|
||||
from lbry.blockchain.lbrycrd import Lbrycrd
|
||||
from lbry.error import LbrycrdEventSubscriptionError
|
||||
|
||||
from . import blocks as block_phase, claims as claim_phase, supports as support_phase
|
||||
from .context import uninitialize
|
||||
from .filter_builder import split_range_into_batches
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
BLOCKS_INIT_EVENT = Event.add("blockchain.sync.blocks.init", "steps")
|
||||
BLOCKS_MAIN_EVENT = Event.add("blockchain.sync.blocks.main", "blocks", "txs")
|
||||
FILTER_INIT_EVENT = Event.add("blockchain.sync.filters.init", "steps")
|
||||
FILTER_MAIN_EVENT = Event.add("blockchain.sync.filters.main", "blocks")
|
||||
CLAIMS_INIT_EVENT = Event.add("blockchain.sync.claims.init", "steps")
|
||||
CLAIMS_MAIN_EVENT = Event.add("blockchain.sync.claims.main", "claims")
|
||||
TRENDS_INIT_EVENT = Event.add("blockchain.sync.trends.init", "steps")
|
||||
TRENDS_MAIN_EVENT = Event.add("blockchain.sync.trends.main", "blocks")
|
||||
SUPPORTS_INIT_EVENT = Event.add("blockchain.sync.supports.init", "steps")
|
||||
SUPPORTS_MAIN_EVENT = Event.add("blockchain.sync.supports.main", "supports")
|
||||
|
||||
|
||||
class BlockchainSync(Sync):
|
||||
|
||||
TX_FLUSH_SIZE = 25_000 # flush to db after processing this many TXs and update progress
|
||||
CLAIM_FLUSH_SIZE = 25_000 # flush to db after processing this many claims and update progress
|
||||
SUPPORT_FLUSH_SIZE = 25_000 # flush to db after processing this many supports and update progress
|
||||
FILTER_FLUSH_SIZE = 10_000 # flush to db after processing this many filters and update progress
|
||||
|
||||
def __init__(self, chain: Lbrycrd, db: Database):
|
||||
super().__init__(chain.ledger, db)
|
||||
self.chain = chain
|
||||
self.pid = os.getpid()
|
||||
self._on_block_controller = EventController()
|
||||
self.on_block = self._on_block_controller.stream
|
||||
self.conf.events.register("blockchain.block", self.on_block)
|
||||
self._on_mempool_controller = EventController()
|
||||
self.on_mempool = self._on_mempool_controller.stream
|
||||
self.on_block_hash_subscription: Optional[BroadcastSubscription] = None
|
||||
self.on_tx_hash_subscription: Optional[BroadcastSubscription] = None
|
||||
self.advance_loop_task: Optional[asyncio.Task] = None
|
||||
self.block_hash_event = asyncio.Event()
|
||||
self.tx_hash_event = asyncio.Event()
|
||||
self.mempool = []
|
||||
self.search_censor_channel_hashes = {
|
||||
unhexlify(channel_id)[::-1] for channel_id in self.conf.search_censor_channel_ids
|
||||
}
|
||||
self.resolve_censor_channel_hashes = {
|
||||
unhexlify(channel_id)[::-1] for channel_id in self.conf.resolve_censor_channel_ids
|
||||
}
|
||||
|
||||
async def wait_for_chain_ready(self):
|
||||
while True:
|
||||
try:
|
||||
return await self.chain.ensure_subscribable()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except LbrycrdEventSubscriptionError as e:
|
||||
log.warning(
|
||||
"Lbrycrd is misconfigured. Please double check if"
|
||||
" zmqpubhashblock is properly set on lbrycrd.conf"
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
log.warning("Blockchain not ready, waiting for it: %s", str(e))
|
||||
await asyncio.sleep(1)
|
||||
|
||||
async def start(self):
|
||||
self.db.stop_event.clear()
|
||||
await self.wait_for_chain_ready()
|
||||
self.advance_loop_task = asyncio.create_task(self.advance())
|
||||
await self.advance_loop_task
|
||||
await self.chain.subscribe()
|
||||
self.advance_loop_task = asyncio.create_task(self.advance_loop())
|
||||
self.on_block_hash_subscription = self.chain.on_block_hash.listen(
|
||||
lambda e: self.block_hash_event.set()
|
||||
)
|
||||
self.on_tx_hash_subscription = self.chain.on_tx_hash.listen(
|
||||
lambda e: self.tx_hash_event.set()
|
||||
)
|
||||
|
||||
async def stop(self):
|
||||
self.chain.unsubscribe()
|
||||
self.db.stop_event.set()
|
||||
for subscription in (
|
||||
self.on_block_hash_subscription,
|
||||
self.on_tx_hash_subscription,
|
||||
self.advance_loop_task
|
||||
):
|
||||
if subscription is not None:
|
||||
subscription.cancel()
|
||||
if isinstance(self.db.executor, ThreadPoolExecutor):
|
||||
await self.db.run(uninitialize)
|
||||
|
||||
async def run_tasks(self, tasks: List[Coroutine]) -> Optional[Set[asyncio.Future]]:
|
||||
done, pending = await asyncio.wait(
|
||||
tasks, return_when=asyncio.FIRST_EXCEPTION
|
||||
)
|
||||
if pending:
|
||||
self.db.stop_event.set()
|
||||
for future in pending:
|
||||
future.cancel()
|
||||
for future in done:
|
||||
future.result()
|
||||
return
|
||||
return done
|
||||
|
||||
async def get_block_headers(self, start_height: int, end_height: int = None):
|
||||
return await self.db.get_block_headers(start_height, end_height)
|
||||
|
||||
async def get_best_block_height(self) -> int:
|
||||
return await self.db.get_best_block_height()
|
||||
|
||||
async def get_best_block_height_for_file(self, file_number) -> int:
|
||||
return await self.db.run(
|
||||
block_phase.get_best_block_height_for_file, file_number
|
||||
)
|
||||
|
||||
async def sync_blocks(self) -> Optional[Tuple[int, int]]:
|
||||
tasks = []
|
||||
starting_height = None
|
||||
tx_count = block_count = 0
|
||||
with Progress(self.db.message_queue, BLOCKS_INIT_EVENT) as p:
|
||||
ending_height = await self.chain.db.get_best_height()
|
||||
for chain_file in p.iter(await self.chain.db.get_block_files()):
|
||||
# block files may be read and saved out of order, need to check
|
||||
# each file individually to see if we have missing blocks
|
||||
our_best_file_height = await self.get_best_block_height_for_file(
|
||||
chain_file['file_number']
|
||||
)
|
||||
if our_best_file_height == chain_file['best_height']:
|
||||
# we have all blocks in this file, skipping
|
||||
continue
|
||||
if -1 < our_best_file_height < chain_file['best_height']:
|
||||
# we have some blocks, need to figure out what we're missing
|
||||
# call get_block_files again limited to this file and current_height
|
||||
chain_file = (await self.chain.db.get_block_files(
|
||||
file_number=chain_file['file_number'], start_height=our_best_file_height+1,
|
||||
))[0]
|
||||
tx_count += chain_file['txs']
|
||||
block_count += chain_file['blocks']
|
||||
file_start_height = chain_file['start_height']
|
||||
starting_height = min(
|
||||
file_start_height if starting_height is None else starting_height,
|
||||
file_start_height
|
||||
)
|
||||
tasks.append(self.db.run(
|
||||
block_phase.sync_block_file, chain_file['file_number'], file_start_height,
|
||||
chain_file['txs'], self.TX_FLUSH_SIZE
|
||||
))
|
||||
with Progress(self.db.message_queue, BLOCKS_MAIN_EVENT) as p:
|
||||
p.start(block_count, tx_count, extra={
|
||||
"starting_height": starting_height,
|
||||
"ending_height": ending_height,
|
||||
"files": len(tasks),
|
||||
"claims": await self.chain.db.get_claim_metadata_count(starting_height, ending_height),
|
||||
"supports": await self.chain.db.get_support_metadata_count(starting_height, ending_height),
|
||||
})
|
||||
completed = await self.run_tasks(tasks)
|
||||
if completed:
|
||||
if starting_height == 0:
|
||||
await self.db.run(block_phase.blocks_constraints_and_indexes)
|
||||
else:
|
||||
await self.db.run(block_phase.blocks_vacuum)
|
||||
best_height_processed = max(f.result() for f in completed)
|
||||
return starting_height, best_height_processed
|
||||
|
||||
async def sync_filters(self):
|
||||
with Progress(self.db.message_queue, FILTER_INIT_EVENT) as p:
|
||||
p.start(2)
|
||||
initial_sync = not await self.db.has_filters()
|
||||
p.step()
|
||||
if initial_sync:
|
||||
blocks = [0, await self.db.get_best_block_height()]
|
||||
else:
|
||||
blocks = await self.db.run(block_phase.get_block_range_without_filters)
|
||||
if blocks != (-1, -1):
|
||||
batches = split_range_into_batches(*blocks)
|
||||
p.step()
|
||||
else:
|
||||
p.step()
|
||||
return
|
||||
with Progress(self.db.message_queue, FILTER_MAIN_EVENT) as p:
|
||||
p.start((blocks[1]-blocks[0])+1)
|
||||
await self.run_tasks([
|
||||
self.db.run(block_phase.sync_filters, *batch) for batch in batches
|
||||
])
|
||||
if initial_sync:
|
||||
await self.db.run(block_phase.filters_constraints_and_indexes)
|
||||
else:
|
||||
await self.db.run(block_phase.filters_vacuum)
|
||||
|
||||
async def sync_spends(self, blocks_added):
|
||||
if blocks_added:
|
||||
await self.db.run(block_phase.sync_spends, blocks_added[0] == 0)
|
||||
|
||||
async def count_unspent_txos(
|
||||
self,
|
||||
txo_types: Tuple[int, ...],
|
||||
blocks: Tuple[int, int] = None,
|
||||
missing_in_supports_table: bool = False,
|
||||
missing_in_claims_table: bool = False,
|
||||
missing_or_stale_in_claims_table: bool = False,
|
||||
) -> int:
|
||||
return await self.db.run(
|
||||
q.count_unspent_txos, txo_types, blocks,
|
||||
missing_in_supports_table,
|
||||
missing_in_claims_table,
|
||||
missing_or_stale_in_claims_table,
|
||||
)
|
||||
|
||||
async def distribute_unspent_txos(
|
||||
self,
|
||||
txo_types: Tuple[int, ...],
|
||||
blocks: Tuple[int, int] = None,
|
||||
missing_in_supports_table: bool = False,
|
||||
missing_in_claims_table: bool = False,
|
||||
missing_or_stale_in_claims_table: bool = False,
|
||||
) -> int:
|
||||
return await self.db.run(
|
||||
q.distribute_unspent_txos, txo_types, blocks,
|
||||
missing_in_supports_table,
|
||||
missing_in_claims_table,
|
||||
missing_or_stale_in_claims_table,
|
||||
self.db.workers
|
||||
)
|
||||
|
||||
async def count_abandoned_supports(self) -> int:
|
||||
return await self.db.run(q.count_abandoned_supports)
|
||||
|
||||
async def count_abandoned_claims(self) -> int:
|
||||
return await self.db.run(q.count_abandoned_claims)
|
||||
|
||||
async def count_claims_with_changed_supports(self, blocks) -> int:
|
||||
return await self.db.run(q.count_claims_with_changed_supports, blocks)
|
||||
|
||||
async def count_claims_with_changed_reposts(self, blocks) -> int:
|
||||
return await self.db.run(q.count_claims_with_changed_reposts, blocks)
|
||||
|
||||
async def count_claims_with_stale_signatures(self, blocks) -> int:
|
||||
return await self.db.run(q.count_claims_with_stale_signatures, blocks)
|
||||
|
||||
async def count_channels_with_changed_content(self, blocks) -> int:
|
||||
return await self.db.run(q.count_channels_with_changed_content, blocks)
|
||||
|
||||
async def count_takeovers(self, blocks) -> int:
|
||||
return await self.chain.db.get_takeover_count(
|
||||
start_height=blocks[0], end_height=blocks[-1]
|
||||
)
|
||||
|
||||
async def sync_claims(self, blocks) -> bool:
|
||||
delete_claims = takeovers = claims_with_changed_supports =\
|
||||
claims_with_changed_reposts = claims_with_stale_signatures = 0
|
||||
initial_sync = not await self.db.has_claims()
|
||||
with Progress(self.db.message_queue, CLAIMS_INIT_EVENT) as p:
|
||||
if initial_sync:
|
||||
total, batches = await self.distribute_unspent_txos(CLAIM_TYPE_CODES)
|
||||
elif blocks:
|
||||
p.start(6)
|
||||
# 1. content claims to be inserted or updated
|
||||
total = await self.count_unspent_txos(
|
||||
CLAIM_TYPE_CODES, blocks, missing_or_stale_in_claims_table=True
|
||||
)
|
||||
batches = [blocks] if total else []
|
||||
p.step()
|
||||
# 2. claims to be deleted
|
||||
delete_claims = await self.count_abandoned_claims()
|
||||
total += delete_claims
|
||||
p.step()
|
||||
# 3. claims to be updated with new support totals
|
||||
claims_with_changed_supports = await self.count_claims_with_changed_supports(blocks)
|
||||
total += claims_with_changed_supports
|
||||
p.step()
|
||||
# 4. claims to be updated with new repost totals
|
||||
claims_with_changed_reposts = await self.count_claims_with_changed_reposts(blocks)
|
||||
total += claims_with_changed_reposts
|
||||
p.step()
|
||||
# 5. claims to be updated due to name takeovers
|
||||
takeovers = await self.count_takeovers(blocks)
|
||||
total += takeovers
|
||||
p.step()
|
||||
# 6. claims where channel signature changed and claim was not re-signed in time
|
||||
claims_with_stale_signatures = await self.count_claims_with_stale_signatures(blocks)
|
||||
total += claims_with_stale_signatures
|
||||
p.step()
|
||||
else:
|
||||
return initial_sync
|
||||
with Progress(self.db.message_queue, CLAIMS_MAIN_EVENT) as p:
|
||||
p.start(total)
|
||||
if batches:
|
||||
await self.run_tasks([
|
||||
self.db.run(claim_phase.claims_insert, batch, not initial_sync, self.CLAIM_FLUSH_SIZE)
|
||||
for batch in batches
|
||||
])
|
||||
if not initial_sync:
|
||||
await self.run_tasks([
|
||||
self.db.run(claim_phase.claims_update, batch) for batch in batches
|
||||
])
|
||||
if delete_claims:
|
||||
await self.db.run(claim_phase.claims_delete, delete_claims)
|
||||
if takeovers:
|
||||
await self.db.run(claim_phase.update_takeovers, blocks, takeovers)
|
||||
if claims_with_changed_supports:
|
||||
await self.db.run(claim_phase.update_stakes, blocks, claims_with_changed_supports)
|
||||
if claims_with_changed_reposts:
|
||||
await self.db.run(claim_phase.update_reposts, blocks, claims_with_changed_reposts)
|
||||
if claims_with_stale_signatures:
|
||||
await self.db.run(claim_phase.update_stale_signatures, blocks, claims_with_stale_signatures)
|
||||
if initial_sync:
|
||||
await self.db.run(claim_phase.claims_constraints_and_indexes)
|
||||
else:
|
||||
await self.db.run(claim_phase.claims_vacuum)
|
||||
return initial_sync
|
||||
|
||||
async def sync_supports(self, blocks):
|
||||
delete_supports = 0
|
||||
initial_sync = not await self.db.has_supports()
|
||||
with Progress(self.db.message_queue, SUPPORTS_INIT_EVENT) as p:
|
||||
if initial_sync:
|
||||
total, support_batches = await self.distribute_unspent_txos(TXO_TYPES['support'])
|
||||
elif blocks:
|
||||
p.start(2)
|
||||
# 1. supports to be inserted
|
||||
total = await self.count_unspent_txos(
|
||||
TXO_TYPES['support'], blocks, missing_in_supports_table=True
|
||||
)
|
||||
support_batches = [blocks] if total else []
|
||||
p.step()
|
||||
# 2. supports to be deleted
|
||||
delete_supports = await self.count_abandoned_supports()
|
||||
total += delete_supports
|
||||
p.step()
|
||||
else:
|
||||
return
|
||||
with Progress(self.db.message_queue, SUPPORTS_MAIN_EVENT) as p:
|
||||
p.start(total)
|
||||
if support_batches:
|
||||
await self.run_tasks([
|
||||
self.db.run(
|
||||
support_phase.supports_insert, batch, not initial_sync, self.SUPPORT_FLUSH_SIZE
|
||||
) for batch in support_batches
|
||||
])
|
||||
if delete_supports:
|
||||
await self.db.run(support_phase.supports_delete, delete_supports)
|
||||
if initial_sync:
|
||||
await self.db.run(support_phase.supports_constraints_and_indexes)
|
||||
else:
|
||||
await self.db.run(support_phase.supports_vacuum)
|
||||
|
||||
async def sync_channel_stats(self, blocks, initial_sync):
|
||||
await self.db.run(claim_phase.update_channel_stats, blocks, initial_sync)
|
||||
|
||||
async def sync_trends(self):
|
||||
ending_height = await self.chain.db.get_best_height()
|
||||
if ending_height is not None:
|
||||
await self.db.run(trending.calculate_trending, ending_height)
|
||||
|
||||
async def sync_claim_filtering(self):
|
||||
await self.db.run(
|
||||
claim_phase.update_claim_filters,
|
||||
self.resolve_censor_channel_hashes,
|
||||
self.search_censor_channel_hashes
|
||||
)
|
||||
|
||||
async def advance(self):
|
||||
blocks_added = await self.sync_blocks()
|
||||
await self.sync_spends(blocks_added)
|
||||
await self.sync_filters()
|
||||
initial_claim_sync = await self.sync_claims(blocks_added)
|
||||
await self.sync_supports(blocks_added)
|
||||
await self.sync_channel_stats(blocks_added, initial_claim_sync)
|
||||
await self.sync_trends()
|
||||
await self.sync_claim_filtering()
|
||||
if blocks_added:
|
||||
await self._on_block_controller.add(BlockEvent(blocks_added[-1]))
|
||||
|
||||
async def sync_mempool(self):
|
||||
added = await self.db.run(block_phase.sync_mempool)
|
||||
await self.sync_spends([-1])
|
||||
await self.db.run(claim_phase.claims_insert, [-1, -1], True, self.CLAIM_FLUSH_SIZE)
|
||||
await self.db.run(claim_phase.claims_update, [-1, -1])
|
||||
await self.db.run(claim_phase.claims_vacuum)
|
||||
self.mempool.extend(added)
|
||||
await self._on_mempool_controller.add(added)
|
||||
|
||||
async def clear_mempool(self):
|
||||
self.mempool.clear()
|
||||
await self.db.run(block_phase.clear_mempool)
|
||||
|
||||
async def advance_loop(self):
|
||||
while True:
|
||||
try:
|
||||
await asyncio.wait([
|
||||
self.tx_hash_event.wait(),
|
||||
self.block_hash_event.wait(),
|
||||
], return_when=asyncio.FIRST_COMPLETED)
|
||||
if self.block_hash_event.is_set():
|
||||
self.block_hash_event.clear()
|
||||
#await self.clear_mempool()
|
||||
await self.advance()
|
||||
self.tx_hash_event.clear()
|
||||
#await self.sync_mempool()
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
await self.stop()
|
||||
|
||||
async def rewind(self, height):
|
||||
await self.db.run(block_phase.rewind, height)
|
||||
@@ -1,837 +0,0 @@
|
||||
import struct
|
||||
import hashlib
|
||||
import logging
|
||||
import asyncio
|
||||
from datetime import date
|
||||
from binascii import hexlify, unhexlify
|
||||
from typing import List, Iterable, Optional, Union
|
||||
|
||||
import ecdsa
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography.hazmat.primitives.serialization import load_der_public_key
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.hazmat.primitives.asymmetric.utils import Prehashed
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
|
||||
from lbry.crypto.hash import hash160, sha256
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.schema.url import normalize_name
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.schema.base import Signable
|
||||
from lbry.schema.purchase import Purchase
|
||||
from lbry.schema.support import Support
|
||||
|
||||
from .script import InputScript, OutputScript
|
||||
from .bcd_data_stream import BCDataStream
|
||||
from .hash import TXRef, TXRefImmutable
|
||||
from .util import ReadOnlyList
|
||||
|
||||
log = logging.getLogger()
|
||||
|
||||
|
||||
class TXRefMutable(TXRef):
|
||||
|
||||
__slots__ = ('tx',)
|
||||
|
||||
def __init__(self, tx: 'Transaction') -> None:
|
||||
super().__init__()
|
||||
self.tx = tx
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
if self._id is None:
|
||||
self._id = hexlify(self.hash[::-1]).decode()
|
||||
return self._id
|
||||
|
||||
@property
|
||||
def hash(self):
|
||||
if self._hash is None:
|
||||
self._hash = sha256(sha256(self.tx.raw_sans_segwit))
|
||||
return self._hash
|
||||
|
||||
@property
|
||||
def height(self):
|
||||
return self.tx.height
|
||||
|
||||
@property
|
||||
def timestamp(self):
|
||||
return self.tx.timestamp
|
||||
|
||||
def reset(self):
|
||||
self._id = None
|
||||
self._hash = None
|
||||
|
||||
|
||||
class TXORef:
|
||||
|
||||
__slots__ = 'tx_ref', 'position'
|
||||
|
||||
def __init__(self, tx_ref: TXRef, position: int) -> None:
|
||||
self.tx_ref = tx_ref
|
||||
self.position = position
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return f'{self.tx_ref.id}:{self.position}'
|
||||
|
||||
@property
|
||||
def hash(self):
|
||||
return self.tx_ref.hash + BCDataStream.uint32.pack(self.position)
|
||||
|
||||
@property
|
||||
def is_null(self):
|
||||
return self.tx_ref.is_null
|
||||
|
||||
@property
|
||||
def txo(self) -> Optional['Output']:
|
||||
return None
|
||||
|
||||
|
||||
class TXORefResolvable(TXORef):
|
||||
|
||||
__slots__ = ('_txo',)
|
||||
|
||||
def __init__(self, txo: 'Output') -> None:
|
||||
assert txo.tx_ref is not None
|
||||
assert txo.position is not None
|
||||
super().__init__(txo.tx_ref, txo.position)
|
||||
self._txo = txo
|
||||
|
||||
@property
|
||||
def txo(self):
|
||||
return self._txo
|
||||
|
||||
|
||||
class InputOutput:
|
||||
|
||||
__slots__ = 'tx_ref', 'position'
|
||||
|
||||
def __init__(self, tx_ref: Union[TXRef, TXRefImmutable] = None, position: int = None) -> None:
|
||||
self.tx_ref = tx_ref
|
||||
self.position = position
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
""" Size of this input / output in bytes. """
|
||||
stream = BCDataStream()
|
||||
self.serialize_to(stream)
|
||||
return len(stream.get_bytes())
|
||||
|
||||
def get_fee(self, ledger):
|
||||
return self.size * ledger.fee_per_byte
|
||||
|
||||
def serialize_to(self, stream, alternate_script=None):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Input(InputOutput):
|
||||
|
||||
NULL_SIGNATURE = b'\x00'*72
|
||||
NULL_PUBLIC_KEY = b'\x00'*33
|
||||
NULL_HASH32 = b'\x00'*32
|
||||
|
||||
__slots__ = 'txo_ref', 'sequence', 'coinbase', 'script'
|
||||
|
||||
def __init__(self, txo_ref: TXORef, script: InputScript, sequence: int = 0xFFFFFFFF,
|
||||
tx_ref: TXRef = None, position: int = None) -> None:
|
||||
super().__init__(tx_ref, position)
|
||||
self.txo_ref = txo_ref
|
||||
self.sequence = sequence
|
||||
self.coinbase = script if txo_ref.is_null else None
|
||||
self.script = script if not txo_ref.is_null else None
|
||||
|
||||
@property
|
||||
def is_coinbase(self):
|
||||
return self.coinbase is not None
|
||||
|
||||
@classmethod
|
||||
def spend(cls, txo: 'Output') -> 'Input':
|
||||
""" Create an input to spend the output."""
|
||||
assert txo.script.is_pay_pubkey_hash, 'Attempting to spend unsupported output.'
|
||||
script = InputScript.redeem_pubkey_hash(cls.NULL_SIGNATURE, cls.NULL_PUBLIC_KEY)
|
||||
return cls(txo.ref, script)
|
||||
|
||||
@classmethod
|
||||
def create_coinbase(cls) -> 'Input':
|
||||
tx_ref = TXRefImmutable.from_hash(cls.NULL_HASH32, 0, 0)
|
||||
txo_ref = TXORef(tx_ref, 0)
|
||||
return cls(txo_ref, b'beef')
|
||||
|
||||
@property
|
||||
def amount(self) -> int:
|
||||
""" Amount this input adds to the transaction. """
|
||||
if self.txo_ref.txo is None:
|
||||
raise ValueError('Cannot resolve output to get amount.')
|
||||
return self.txo_ref.txo.amount
|
||||
|
||||
@property
|
||||
def is_my_input(self) -> Optional[bool]:
|
||||
""" True if the output this input spends is yours. """
|
||||
if self.txo_ref.txo is None:
|
||||
return False
|
||||
return self.txo_ref.txo.is_my_output
|
||||
|
||||
@classmethod
|
||||
def deserialize_from(cls, stream):
|
||||
tx_ref = TXRefImmutable.from_hash(stream.read(32), -1, -1)
|
||||
position = stream.read_uint32()
|
||||
script = stream.read_string()
|
||||
sequence = stream.read_uint32()
|
||||
return cls(
|
||||
TXORef(tx_ref, position),
|
||||
InputScript(script) if not tx_ref.is_null else script,
|
||||
sequence
|
||||
)
|
||||
|
||||
def serialize_to(self, stream, alternate_script=None):
|
||||
stream.write(self.txo_ref.tx_ref.hash)
|
||||
stream.write_uint32(self.txo_ref.position)
|
||||
if alternate_script is not None:
|
||||
stream.write_string(alternate_script)
|
||||
else:
|
||||
if self.is_coinbase:
|
||||
stream.write_string(self.coinbase)
|
||||
else:
|
||||
stream.write_string(self.script.source)
|
||||
stream.write_uint32(self.sequence)
|
||||
|
||||
|
||||
class Output(InputOutput):
|
||||
|
||||
__slots__ = (
|
||||
'amount', 'script', 'is_internal_transfer', 'spent_height', 'is_my_output', 'is_my_input',
|
||||
'channel', 'private_key', 'meta', 'sent_supports', 'sent_tips', 'received_tips',
|
||||
'purchase', 'purchased_claim', 'purchase_receipt',
|
||||
'reposted_claim', 'claims', '_signable'
|
||||
)
|
||||
|
||||
def __init__(self, amount: int, script: OutputScript,
|
||||
tx_ref: TXRef = None, position: int = None,
|
||||
is_internal_transfer: Optional[bool] = None, spent_height: Optional[bool] = None,
|
||||
is_my_output: Optional[bool] = None, is_my_input: Optional[bool] = None,
|
||||
sent_supports: Optional[int] = None, sent_tips: Optional[int] = None,
|
||||
received_tips: Optional[int] = None,
|
||||
channel: Optional['Output'] = None, private_key: Optional[str] = None
|
||||
) -> None:
|
||||
super().__init__(tx_ref, position)
|
||||
self.amount = amount
|
||||
self.script = script
|
||||
self.is_internal_transfer = is_internal_transfer
|
||||
self.spent_height = spent_height
|
||||
self.is_my_output = is_my_output
|
||||
self.is_my_input = is_my_input
|
||||
self.sent_supports = sent_supports
|
||||
self.sent_tips = sent_tips
|
||||
self.received_tips = received_tips
|
||||
self.channel = channel
|
||||
self.private_key = private_key
|
||||
self.purchase: 'Output' = None # txo containing purchase metadata
|
||||
self.purchased_claim: 'Output' = None # resolved claim pointed to by purchase
|
||||
self.purchase_receipt: 'Output' = None # txo representing purchase receipt for this claim
|
||||
self.reposted_claim: 'Output' = None # txo representing claim being reposted
|
||||
self.claims: List['Output'] = None # resolved claims for collection
|
||||
self._signable: Optional[Signable] = None
|
||||
self.meta = {}
|
||||
|
||||
def update_annotations(self, annotated: 'Output'):
|
||||
if annotated is None:
|
||||
self.is_internal_transfer = None
|
||||
self.spent_height = None
|
||||
self.is_my_output = None
|
||||
self.is_my_input = None
|
||||
self.sent_supports = None
|
||||
self.sent_tips = None
|
||||
self.received_tips = None
|
||||
else:
|
||||
self.is_internal_transfer = annotated.is_internal_transfer
|
||||
self.spent_height = annotated.spent_height
|
||||
self.is_my_output = annotated.is_my_output
|
||||
self.is_my_input = annotated.is_my_input
|
||||
self.sent_supports = annotated.sent_supports
|
||||
self.sent_tips = annotated.sent_tips
|
||||
self.received_tips = annotated.received_tips
|
||||
self.channel = annotated.channel if annotated else None
|
||||
self.private_key = annotated.private_key if annotated else None
|
||||
|
||||
@property
|
||||
def ref(self):
|
||||
return TXORefResolvable(self)
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self.ref.id
|
||||
|
||||
@property
|
||||
def hash(self):
|
||||
return self.ref.hash
|
||||
|
||||
@property
|
||||
def is_spent(self):
|
||||
if self.spent_height is not None:
|
||||
return self.spent_height > 0
|
||||
|
||||
@property
|
||||
def pubkey_hash(self):
|
||||
pubkey_hash = self.script.values.get('pubkey_hash')
|
||||
if pubkey_hash:
|
||||
return pubkey_hash
|
||||
return hash160(self.script.values['pubkey'])
|
||||
|
||||
@property
|
||||
def has_address(self):
|
||||
return (
|
||||
'pubkey_hash' in self.script.values or
|
||||
'script_hash' in self.script.values or
|
||||
'pubkey' in self.script.values
|
||||
)
|
||||
|
||||
def get_address(self, ledger):
|
||||
if 'script_hash' in self.script.values:
|
||||
return ledger.script_hash_to_address(self.script.values['script_hash'])
|
||||
return ledger.pubkey_hash_to_address(self.pubkey_hash)
|
||||
|
||||
@classmethod
|
||||
def pay_pubkey_hash(cls, amount, pubkey_hash):
|
||||
return cls(amount, OutputScript.pay_pubkey_hash(pubkey_hash))
|
||||
|
||||
@classmethod
|
||||
def deserialize_from(cls, stream, transaction_offset: int = 0):
|
||||
amount = stream.read_uint64()
|
||||
length = stream.read_compact_size()
|
||||
offset = stream.tell()-transaction_offset
|
||||
script = OutputScript(stream.read(length), offset=offset)
|
||||
return cls(amount=amount, script=script)
|
||||
|
||||
def serialize_to(self, stream, alternate_script=None):
|
||||
stream.write_uint64(self.amount)
|
||||
stream.write_string(self.script.source)
|
||||
|
||||
def get_fee(self, ledger):
|
||||
name_fee = 0
|
||||
if self.script.is_claim_name:
|
||||
name_fee = len(self.script.values['claim_name']) * ledger.fee_per_name_char
|
||||
return max(name_fee, super().get_fee(ledger))
|
||||
|
||||
@property
|
||||
def is_claim(self) -> bool:
|
||||
return self.script.is_claim_name or self.script.is_update_claim
|
||||
|
||||
@property
|
||||
def is_support(self) -> bool:
|
||||
return self.script.is_support_claim
|
||||
|
||||
@property
|
||||
def is_support_data(self) -> bool:
|
||||
return self.script.is_support_claim_data
|
||||
|
||||
@property
|
||||
def claim_hash(self) -> bytes:
|
||||
if self.script.is_claim_name:
|
||||
return hash160(self.tx_ref.hash + struct.pack('>I', self.position))
|
||||
elif self.script.is_update_claim or self.script.is_support_claim:
|
||||
return self.script.values['claim_id']
|
||||
else:
|
||||
raise ValueError('No claim_id associated.')
|
||||
|
||||
@property
|
||||
def claim_id(self) -> str:
|
||||
return hexlify(self.claim_hash[::-1]).decode()
|
||||
|
||||
@property
|
||||
def claim_name(self) -> str:
|
||||
if self.script.is_claim_involved:
|
||||
return self.script.values['claim_name'].decode()
|
||||
raise ValueError('No claim_name associated.')
|
||||
|
||||
@property
|
||||
def normalized_name(self) -> str:
|
||||
return normalize_name(self.claim_name)
|
||||
|
||||
@property
|
||||
def claim(self) -> Claim:
|
||||
if self.is_claim:
|
||||
if not isinstance(self.script.values['claim'], Claim):
|
||||
self.script.values['claim'] = Claim.from_bytes(self.script.values['claim'])
|
||||
return self.script.values['claim']
|
||||
raise ValueError('Only claim name and claim update have the claim payload.')
|
||||
|
||||
@property
|
||||
def can_decode_claim(self):
|
||||
try:
|
||||
return self.claim
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@property
|
||||
def support(self) -> Support:
|
||||
if self.is_support_data:
|
||||
if not isinstance(self.script.values['support'], Support):
|
||||
self.script.values['support'] = Support.from_bytes(self.script.values['support'])
|
||||
return self.script.values['support']
|
||||
raise ValueError('Only supports with data can be represented as Supports.')
|
||||
|
||||
@property
|
||||
def can_decode_support(self):
|
||||
try:
|
||||
return self.support
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@property
|
||||
def signable(self) -> Signable:
|
||||
if self._signable is None:
|
||||
if self.is_claim:
|
||||
self._signable = self.claim
|
||||
elif self.is_support_data:
|
||||
self._signable = self.support
|
||||
return self._signable
|
||||
|
||||
@property
|
||||
def can_decode_signable(self) -> Signable:
|
||||
try:
|
||||
return self.signable
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@property
|
||||
def permanent_url(self) -> str:
|
||||
if self.script.is_claim_involved:
|
||||
return f"lbry://{self.claim_name}#{self.claim_id}"
|
||||
raise ValueError('No claim associated.')
|
||||
|
||||
@property
|
||||
def has_private_key(self):
|
||||
return self.private_key is not None
|
||||
|
||||
def get_signature_digest(self, ledger):
|
||||
if self.signable.unsigned_payload:
|
||||
pieces = [
|
||||
Base58.decode(self.get_address(ledger)),
|
||||
self.signable.unsigned_payload,
|
||||
self.signable.signing_channel_hash[::-1]
|
||||
]
|
||||
else:
|
||||
pieces = [
|
||||
self.tx_ref.tx.inputs[0].txo_ref.hash,
|
||||
self.signable.signing_channel_hash,
|
||||
self.signable.to_message_bytes()
|
||||
]
|
||||
return sha256(b''.join(pieces))
|
||||
|
||||
def get_encoded_signature(self):
|
||||
signature = hexlify(self.signable.signature)
|
||||
r = int(signature[:int(len(signature)/2)], 16)
|
||||
s = int(signature[int(len(signature)/2):], 16)
|
||||
return ecdsa.util.sigencode_der(r, s, len(signature)*4)
|
||||
|
||||
@staticmethod
|
||||
def is_signature_valid(encoded_signature, signature_digest, public_key_bytes):
|
||||
try:
|
||||
public_key = load_der_public_key(public_key_bytes, default_backend())
|
||||
public_key.verify( # pylint: disable=no-value-for-parameter
|
||||
encoded_signature, signature_digest,
|
||||
ec.ECDSA(Prehashed(hashes.SHA256()))
|
||||
)
|
||||
return True
|
||||
except (ValueError, InvalidSignature):
|
||||
pass
|
||||
return False
|
||||
|
||||
def is_signed_by(self, channel: 'Output', ledger=None):
|
||||
return self.is_signature_valid(
|
||||
self.get_encoded_signature(),
|
||||
self.get_signature_digest(ledger),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
|
||||
def sign(self, channel: 'Output', first_input_id=None):
|
||||
self.channel = channel
|
||||
self.signable.signing_channel_hash = channel.claim_hash
|
||||
digest = sha256(b''.join([
|
||||
first_input_id or self.tx_ref.tx.inputs[0].txo_ref.hash,
|
||||
self.signable.signing_channel_hash,
|
||||
self.signable.to_message_bytes()
|
||||
]))
|
||||
self.signable.signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
self.script.generate()
|
||||
|
||||
def clear_signature(self):
|
||||
self.channel = None
|
||||
self.claim.clear_signature()
|
||||
|
||||
@staticmethod
|
||||
def _sync_generate_channel_private_key():
|
||||
private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256)
|
||||
public_key_bytes = private_key.get_verifying_key().to_der()
|
||||
return private_key, public_key_bytes
|
||||
|
||||
async def generate_channel_private_key(self):
|
||||
private_key, public_key_bytes = await asyncio.get_running_loop().run_in_executor(
|
||||
None, Output._sync_generate_channel_private_key
|
||||
)
|
||||
self.private_key = private_key
|
||||
self.claim.channel.public_key_bytes = public_key_bytes
|
||||
self.script.generate()
|
||||
return self.private_key
|
||||
|
||||
def is_channel_private_key(self, private_key):
|
||||
return self.claim.channel.public_key_bytes == private_key.get_verifying_key().to_der()
|
||||
|
||||
@classmethod
|
||||
def pay_claim_name_pubkey_hash(
|
||||
cls, amount: int, claim_name: str, claim: Claim, pubkey_hash: bytes) -> 'Output':
|
||||
script = OutputScript.pay_claim_name_pubkey_hash(
|
||||
claim_name.encode(), claim, pubkey_hash)
|
||||
return cls(amount, script)
|
||||
|
||||
@classmethod
|
||||
def pay_update_claim_pubkey_hash(
|
||||
cls, amount: int, claim_name: str, claim_id: str, claim: Claim, pubkey_hash: bytes) -> 'Output':
|
||||
script = OutputScript.pay_update_claim_pubkey_hash(
|
||||
claim_name.encode(), unhexlify(claim_id)[::-1], claim, pubkey_hash
|
||||
)
|
||||
return cls(amount, script)
|
||||
|
||||
@classmethod
|
||||
def pay_support_pubkey_hash(cls, amount: int, claim_name: str, claim_id: str, pubkey_hash: bytes) -> 'Output':
|
||||
script = OutputScript.pay_support_pubkey_hash(
|
||||
claim_name.encode(), unhexlify(claim_id)[::-1], pubkey_hash
|
||||
)
|
||||
return cls(amount, script)
|
||||
|
||||
@classmethod
|
||||
def pay_support_data_pubkey_hash(
|
||||
cls, amount: int, claim_name: str, claim_id: str, support: Support, pubkey_hash: bytes) -> 'Output':
|
||||
script = OutputScript.pay_support_data_pubkey_hash(
|
||||
claim_name.encode(), unhexlify(claim_id)[::-1], support, pubkey_hash
|
||||
)
|
||||
return cls(amount, script)
|
||||
|
||||
@classmethod
|
||||
def add_purchase_data(cls, purchase: Purchase) -> 'Output':
|
||||
script = OutputScript.return_data(purchase)
|
||||
return cls(0, script)
|
||||
|
||||
@property
|
||||
def is_purchase_data(self) -> bool:
|
||||
return self.script.is_return_data and (
|
||||
isinstance(self.script.values['data'], Purchase) or
|
||||
Purchase.has_start_byte(self.script.values['data'])
|
||||
)
|
||||
|
||||
@property
|
||||
def purchase_data(self) -> Purchase:
|
||||
if self.is_purchase_data:
|
||||
if not isinstance(self.script.values['data'], Purchase):
|
||||
self.script.values['data'] = Purchase.from_bytes(self.script.values['data'])
|
||||
return self.script.values['data']
|
||||
raise ValueError('Output does not have purchase data.')
|
||||
|
||||
@property
|
||||
def can_decode_purchase_data(self):
|
||||
try:
|
||||
return self.purchase_data
|
||||
except: # pylint: disable=bare-except
|
||||
return False
|
||||
|
||||
@property
|
||||
def purchased_claim_id(self):
|
||||
if self.purchase is not None:
|
||||
return self.purchase.purchase_data.claim_id
|
||||
if self.purchased_claim is not None:
|
||||
return self.purchased_claim.claim_id
|
||||
|
||||
@property
|
||||
def purchased_claim_hash(self):
|
||||
if self.purchase is not None:
|
||||
return self.purchase.purchase_data.claim_hash
|
||||
if self.purchased_claim is not None:
|
||||
return self.purchased_claim.claim_hash
|
||||
|
||||
@property
|
||||
def has_price(self):
|
||||
if self.can_decode_claim:
|
||||
claim = self.claim
|
||||
if claim.is_stream:
|
||||
stream = claim.stream
|
||||
return stream.has_fee and stream.fee.amount and stream.fee.amount > 0
|
||||
return False
|
||||
|
||||
@property
|
||||
def price(self):
|
||||
return self.claim.stream.fee
|
||||
|
||||
|
||||
class Transaction:
|
||||
|
||||
def __init__(self, raw=None, version: int = 1, locktime: int = 0, is_verified: bool = False,
|
||||
height: int = -2, position: int = -1, timestamp: int = 0) -> None:
|
||||
self._raw = raw
|
||||
self._raw_sans_segwit = None
|
||||
self.is_segwit_flag = 0
|
||||
self.witnesses: List[bytes] = []
|
||||
self.ref = TXRefMutable(self)
|
||||
self.version = version
|
||||
self.locktime = locktime
|
||||
self._inputs: List[Input] = []
|
||||
self._outputs: List[Output] = []
|
||||
self.is_verified = is_verified
|
||||
# Height Progression
|
||||
# -2: not broadcast
|
||||
# -1: in mempool but has unconfirmed inputs
|
||||
# 0: in mempool and all inputs confirmed
|
||||
# +num: confirmed in a specific block (height)
|
||||
self.height = height
|
||||
self.position = position
|
||||
self.timestamp = timestamp
|
||||
self._day: int = 0
|
||||
if raw is not None:
|
||||
self.deserialize()
|
||||
|
||||
def __repr__(self):
|
||||
return f"TX({self.id[:10]}...{self.id[-10:]})"
|
||||
|
||||
@property
|
||||
def is_broadcast(self):
|
||||
return self.height > -2
|
||||
|
||||
@property
|
||||
def is_mempool(self):
|
||||
return self.height in (-1, 0)
|
||||
|
||||
@property
|
||||
def is_confirmed(self):
|
||||
return self.height > 0
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self.ref.id
|
||||
|
||||
@property
|
||||
def hash(self):
|
||||
return self.ref.hash
|
||||
|
||||
@property
|
||||
def day(self):
|
||||
if self._day is None and self.timestamp > 0:
|
||||
self._day = date.fromtimestamp(self.timestamp).toordinal()
|
||||
return self._day
|
||||
|
||||
@property
|
||||
def raw(self):
|
||||
if self._raw is None:
|
||||
self._raw = self._serialize()
|
||||
return self._raw
|
||||
|
||||
@property
|
||||
def raw_sans_segwit(self):
|
||||
if self.is_segwit_flag:
|
||||
if self._raw_sans_segwit is None:
|
||||
self._raw_sans_segwit = self._serialize(sans_segwit=True)
|
||||
return self._raw_sans_segwit
|
||||
return self.raw
|
||||
|
||||
def _reset(self):
|
||||
self._raw = None
|
||||
self._raw_sans_segwit = None
|
||||
self.ref.reset()
|
||||
|
||||
@property
|
||||
def inputs(self) -> ReadOnlyList[Input]:
|
||||
return ReadOnlyList(self._inputs)
|
||||
|
||||
@property
|
||||
def outputs(self) -> ReadOnlyList[Output]:
|
||||
return ReadOnlyList(self._outputs)
|
||||
|
||||
def _add(self, existing_ios: List, new_ios: Iterable[InputOutput], reset=False) -> 'Transaction':
|
||||
for txio in new_ios:
|
||||
txio.tx_ref = self.ref
|
||||
txio.position = len(existing_ios)
|
||||
existing_ios.append(txio)
|
||||
if reset:
|
||||
self._reset()
|
||||
return self
|
||||
|
||||
def add_inputs(self, inputs: Iterable[Input]) -> 'Transaction':
|
||||
return self._add(self._inputs, inputs, True)
|
||||
|
||||
def add_outputs(self, outputs: Iterable[Output]) -> 'Transaction':
|
||||
return self._add(self._outputs, outputs, True)
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
""" Size in bytes of the entire transaction. """
|
||||
return len(self.raw)
|
||||
|
||||
@property
|
||||
def base_size(self) -> int:
|
||||
""" Size of transaction without inputs or outputs in bytes. """
|
||||
return (
|
||||
self.size
|
||||
- sum(txi.size for txi in self._inputs)
|
||||
- sum(txo.size for txo in self._outputs)
|
||||
)
|
||||
|
||||
@property
|
||||
def input_sum(self):
|
||||
return sum(i.amount for i in self.inputs if i.txo_ref.txo is not None)
|
||||
|
||||
@property
|
||||
def output_sum(self):
|
||||
return sum(o.amount for o in self.outputs)
|
||||
|
||||
@property
|
||||
def net_account_balance(self) -> int:
|
||||
balance = 0
|
||||
for txi in self.inputs:
|
||||
if txi.txo_ref.txo is None:
|
||||
continue
|
||||
if txi.is_my_input is True:
|
||||
balance -= txi.amount
|
||||
elif txi.is_my_input is None:
|
||||
raise ValueError(
|
||||
"Cannot access net_account_balance if inputs do not "
|
||||
"have is_my_input set properly."
|
||||
)
|
||||
for txo in self.outputs:
|
||||
if txo.is_my_output is True:
|
||||
balance += txo.amount
|
||||
elif txo.is_my_output is None:
|
||||
raise ValueError(
|
||||
"Cannot access net_account_balance if outputs do not "
|
||||
"have is_my_output set properly."
|
||||
)
|
||||
return balance
|
||||
|
||||
@property
|
||||
def fee(self) -> int:
|
||||
return self.input_sum - self.output_sum
|
||||
|
||||
def get_base_fee(self, ledger) -> int:
|
||||
""" Fee for base tx excluding inputs and outputs. """
|
||||
return self.base_size * ledger.fee_per_byte
|
||||
|
||||
def get_effective_input_sum(self, ledger) -> int:
|
||||
""" Sum of input values *minus* the cost involved to spend them. """
|
||||
return sum(txi.amount - txi.get_fee(ledger) for txi in self._inputs)
|
||||
|
||||
def get_total_output_sum(self, ledger) -> int:
|
||||
""" Sum of output values *plus* the cost involved to spend them. """
|
||||
return sum(txo.amount + txo.get_fee(ledger) for txo in self._outputs)
|
||||
|
||||
def _serialize(self, with_inputs: bool = True, sans_segwit: bool = False) -> bytes:
|
||||
stream = BCDataStream()
|
||||
stream.write_uint32(self.version)
|
||||
if with_inputs:
|
||||
stream.write_compact_size(len(self._inputs))
|
||||
for txin in self._inputs:
|
||||
txin.serialize_to(stream)
|
||||
stream.write_compact_size(len(self._outputs))
|
||||
for txout in self._outputs:
|
||||
txout.serialize_to(stream)
|
||||
stream.write_uint32(self.locktime)
|
||||
return stream.get_bytes()
|
||||
|
||||
def _serialize_for_signature(self, signing_input: int) -> bytes:
|
||||
stream = BCDataStream()
|
||||
stream.write_uint32(self.version)
|
||||
stream.write_compact_size(len(self._inputs))
|
||||
for i, txin in enumerate(self._inputs):
|
||||
if signing_input == i:
|
||||
assert txin.txo_ref.txo is not None
|
||||
txin.serialize_to(stream, txin.txo_ref.txo.script.source)
|
||||
else:
|
||||
txin.serialize_to(stream, b'')
|
||||
stream.write_compact_size(len(self._outputs))
|
||||
for txout in self._outputs:
|
||||
txout.serialize_to(stream)
|
||||
stream.write_uint32(self.locktime)
|
||||
stream.write_uint32(self.signature_hash_type(1)) # signature hash type: SIGHASH_ALL
|
||||
return stream.get_bytes()
|
||||
|
||||
def deserialize(self, stream=None):
|
||||
if self._raw is not None or stream is not None:
|
||||
stream = stream or BCDataStream(self._raw)
|
||||
start = stream.tell()
|
||||
self.version = stream.read_uint32()
|
||||
input_count = stream.read_compact_size()
|
||||
if input_count == 0:
|
||||
self.is_segwit_flag = stream.read_uint8()
|
||||
input_count = stream.read_compact_size()
|
||||
self._add(self._inputs, [
|
||||
Input.deserialize_from(stream) for _ in range(input_count)
|
||||
])
|
||||
output_count = stream.read_compact_size()
|
||||
self._add(self._outputs, [
|
||||
Output.deserialize_from(stream, start) for _ in range(output_count)
|
||||
])
|
||||
if self.is_segwit_flag:
|
||||
# drain witness portion of transaction
|
||||
# too many witnesses for no crime
|
||||
self.witnesses = []
|
||||
for _ in range(input_count):
|
||||
for _ in range(stream.read_compact_size()):
|
||||
self.witnesses.append(stream.read(stream.read_compact_size()))
|
||||
self.locktime = stream.read_uint32()
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def signature_hash_type(hash_type):
|
||||
return hash_type
|
||||
|
||||
@property
|
||||
def my_inputs(self):
|
||||
for txi in self.inputs:
|
||||
if txi.txo_ref.txo is not None and txi.txo_ref.txo.is_my_output:
|
||||
yield txi
|
||||
|
||||
def _filter_my_outputs(self, f):
|
||||
for txo in self.outputs:
|
||||
if txo.is_my_output and f(txo.script):
|
||||
yield txo
|
||||
|
||||
def _filter_other_outputs(self, f):
|
||||
for txo in self.outputs:
|
||||
if not txo.is_my_output and f(txo.script):
|
||||
yield txo
|
||||
|
||||
def _filter_any_outputs(self, f):
|
||||
for txo in self.outputs:
|
||||
if f(txo):
|
||||
yield txo
|
||||
|
||||
@property
|
||||
def my_claim_outputs(self):
|
||||
return self._filter_my_outputs(lambda s: s.is_claim_name)
|
||||
|
||||
@property
|
||||
def my_update_outputs(self):
|
||||
return self._filter_my_outputs(lambda s: s.is_update_claim)
|
||||
|
||||
@property
|
||||
def my_support_outputs(self):
|
||||
return self._filter_my_outputs(lambda s: s.is_support_claim)
|
||||
|
||||
@property
|
||||
def any_purchase_outputs(self):
|
||||
return self._filter_any_outputs(lambda o: o.purchase is not None)
|
||||
|
||||
@property
|
||||
def other_support_outputs(self):
|
||||
return self._filter_other_outputs(lambda s: s.is_support_claim)
|
||||
|
||||
@property
|
||||
def my_abandon_outputs(self):
|
||||
for txi in self.inputs:
|
||||
abandon = txi.txo_ref.txo
|
||||
if abandon is not None and abandon.is_my_output and abandon.script.is_claim_involved:
|
||||
is_update = False
|
||||
if abandon.script.is_claim_name or abandon.script.is_update_claim:
|
||||
for update in self.my_update_outputs:
|
||||
if abandon.claim_id == update.claim_id:
|
||||
is_update = True
|
||||
break
|
||||
if not is_update:
|
||||
yield abandon
|
||||
@@ -1,93 +0,0 @@
|
||||
from typing import TypeVar, Sequence, Optional
|
||||
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
|
||||
class ReadOnlyList(Sequence[T]):
|
||||
|
||||
def __init__(self, lst):
|
||||
self.lst = lst
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.lst[key]
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.lst)
|
||||
|
||||
|
||||
class ArithUint256:
|
||||
# https://github.com/bitcoin/bitcoin/blob/master/src/arith_uint256.cpp
|
||||
|
||||
__slots__ = '_value', '_compact'
|
||||
|
||||
def __init__(self, value: int) -> None:
|
||||
self._value = value
|
||||
self._compact: Optional[int] = None
|
||||
|
||||
@classmethod
|
||||
def from_compact(cls, compact) -> 'ArithUint256':
|
||||
size = compact >> 24
|
||||
word = compact & 0x007fffff
|
||||
if size <= 3:
|
||||
return cls(word >> 8 * (3 - size))
|
||||
else:
|
||||
return cls(word << 8 * (size - 3))
|
||||
|
||||
@property
|
||||
def value(self) -> int:
|
||||
return self._value
|
||||
|
||||
@property
|
||||
def compact(self) -> int:
|
||||
if self._compact is None:
|
||||
self._compact = self._calculate_compact()
|
||||
return self._compact
|
||||
|
||||
@property
|
||||
def negative(self) -> int:
|
||||
return self._calculate_compact(negative=True)
|
||||
|
||||
@property
|
||||
def bits(self) -> int:
|
||||
""" Returns the position of the highest bit set plus one. """
|
||||
bits = bin(self._value)[2:]
|
||||
for i, d in enumerate(bits):
|
||||
if d:
|
||||
return (len(bits) - i) + 1
|
||||
return 0
|
||||
|
||||
@property
|
||||
def low64(self) -> int:
|
||||
return self._value & 0xffffffffffffffff
|
||||
|
||||
def _calculate_compact(self, negative=False) -> int:
|
||||
size = (self.bits + 7) // 8
|
||||
if size <= 3:
|
||||
compact = self.low64 << 8 * (3 - size)
|
||||
else:
|
||||
compact = ArithUint256(self._value >> 8 * (size - 3)).low64
|
||||
# The 0x00800000 bit denotes the sign.
|
||||
# Thus, if it is already set, divide the mantissa by 256 and increase the exponent.
|
||||
if compact & 0x00800000:
|
||||
compact >>= 8
|
||||
size += 1
|
||||
assert (compact & ~0x007fffff) == 0
|
||||
assert size < 256
|
||||
compact |= size << 24
|
||||
if negative and compact & 0x007fffff:
|
||||
compact |= 0x00800000
|
||||
return compact
|
||||
|
||||
def __mul__(self, x):
|
||||
# Take the mod because we are limited to an unsigned 256 bit number
|
||||
return ArithUint256((self._value * x) % 2 ** 256)
|
||||
|
||||
def __truediv__(self, x):
|
||||
return ArithUint256(int(self._value / x))
|
||||
|
||||
def __gt__(self, other):
|
||||
return self._value > other
|
||||
|
||||
def __lt__(self, other):
|
||||
return self._value < other
|
||||
@@ -1,4 +0,0 @@
|
||||
# don't touch this. CI server changes this during build/deployment
|
||||
BUILD = "dev"
|
||||
COMMIT_HASH = "none"
|
||||
DOCKER_TAG = "none"
|
||||
-283
@@ -1,283 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
import pathlib
|
||||
import argparse
|
||||
import textwrap
|
||||
import subprocess
|
||||
|
||||
from docopt import docopt
|
||||
|
||||
from lbry import __version__
|
||||
from lbry.conf import Config, CLIConfig
|
||||
from lbry.service import Daemon, Client, FullNode, LightClient
|
||||
from lbry.service.metadata import interface
|
||||
|
||||
|
||||
def split_subparser_argument(parent, original, name, condition):
|
||||
new_sub_parser = argparse._SubParsersAction(
|
||||
original.option_strings,
|
||||
original._prog_prefix,
|
||||
original._parser_class,
|
||||
metavar=original.metavar
|
||||
)
|
||||
new_sub_parser._name_parser_map = original._name_parser_map
|
||||
new_sub_parser._choices_actions = [
|
||||
a for a in original._choices_actions if condition(original._name_parser_map[a.dest])
|
||||
]
|
||||
group = argparse._ArgumentGroup(parent, name)
|
||||
group._group_actions = [new_sub_parser]
|
||||
return group
|
||||
|
||||
|
||||
class ArgumentParser(argparse.ArgumentParser):
|
||||
def __init__(self, *args, group_name=None, **kwargs):
|
||||
super().__init__(*args, formatter_class=HelpFormatter, add_help=False, **kwargs)
|
||||
self.add_argument(
|
||||
'--help', dest='help', action='store_true', default=False,
|
||||
help='Show this help message and exit.'
|
||||
)
|
||||
self._optionals.title = 'Options'
|
||||
if group_name is None:
|
||||
self.epilog = (
|
||||
"Run 'lbrynet COMMAND --help' for more information on a command or group."
|
||||
)
|
||||
else:
|
||||
self.epilog = (
|
||||
f"Run 'lbrynet {group_name} COMMAND --help' for more information on a command."
|
||||
)
|
||||
self.set_defaults(group=group_name, group_parser=self)
|
||||
|
||||
def format_help(self):
|
||||
formatter = self._get_formatter()
|
||||
formatter.add_usage(
|
||||
self.usage, self._actions, self._mutually_exclusive_groups
|
||||
)
|
||||
formatter.add_text(self.description)
|
||||
|
||||
# positionals, optionals and user-defined groups
|
||||
for action_group in self._granular_action_groups:
|
||||
formatter.start_section(action_group.title)
|
||||
formatter.add_text(action_group.description)
|
||||
formatter.add_arguments(action_group._group_actions)
|
||||
formatter.end_section()
|
||||
|
||||
formatter.add_text(self.epilog)
|
||||
return formatter.format_help()
|
||||
|
||||
@property
|
||||
def _granular_action_groups(self):
|
||||
if self.prog != 'lbrynet':
|
||||
yield from self._action_groups
|
||||
return
|
||||
yield self._optionals
|
||||
action: argparse._SubParsersAction = self._positionals._group_actions[0]
|
||||
yield split_subparser_argument(
|
||||
self, action, "Grouped Commands", lambda parser: 'group' in parser._defaults
|
||||
)
|
||||
yield split_subparser_argument(
|
||||
self, action, "Commands", lambda parser: 'group' not in parser._defaults
|
||||
)
|
||||
|
||||
def error(self, message):
|
||||
self.print_help(argparse._sys.stderr)
|
||||
self.exit(2, f"\n{message}\n")
|
||||
|
||||
|
||||
class HelpFormatter(argparse.HelpFormatter):
|
||||
|
||||
def add_usage(self, usage, actions, groups, prefix='Usage: '):
|
||||
super().add_usage(
|
||||
usage, [a for a in actions if a.option_strings != ['--help']], groups, prefix
|
||||
)
|
||||
|
||||
|
||||
def add_command_parser(parent, method_name, command):
|
||||
short = command['desc']['text'][0] if command['desc'] else ''
|
||||
subcommand = parent.add_parser(command['name'], help=short)
|
||||
subcommand.set_defaults(api_method_name=method_name, command=command['name'], doc=command['help'])
|
||||
|
||||
|
||||
def get_argument_parser():
|
||||
root = ArgumentParser(
|
||||
'lbrynet', description='An interface to the LBRY Network.', allow_abbrev=False,
|
||||
)
|
||||
root.add_argument(
|
||||
'-v', '--version', dest='cli_version', action="store_true",
|
||||
help='Show lbrynet CLI version and exit.'
|
||||
)
|
||||
root.set_defaults(group=None, command=None)
|
||||
CLIConfig.contribute_to_argparse(root)
|
||||
sub = root.add_subparsers(metavar='COMMAND')
|
||||
start = sub.add_parser(
|
||||
'start',
|
||||
usage='lbrynet start [--config FILE] [--data-dir DIR] [--wallet-dir DIR] [--download-dir DIR] ...',
|
||||
help='Start LBRY Network interface.'
|
||||
)
|
||||
start.add_argument(
|
||||
"service", choices=[LightClient.name, FullNode.name], default=LightClient.name, nargs="?"
|
||||
)
|
||||
start.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--no-logging', dest='no_logging', action="store_true",
|
||||
help='Disable all logging of any kind.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output for lbry logger and event loop. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
start.add_argument(
|
||||
'--initial-headers', dest='initial_headers',
|
||||
help='Specify path to initial blockchain headers, faster than downloading them on first run.'
|
||||
)
|
||||
install = sub.add_parser("install", help="Install lbrynet with various system services.")
|
||||
install.add_argument("system", choices=["systemd"])
|
||||
install.add_argument(
|
||||
"--global", dest="install_global", action="store_true",
|
||||
help="Install system wide (requires running as root), default is for current user only."
|
||||
)
|
||||
Config.contribute_to_argparse(start)
|
||||
start.set_defaults(command='start', start_parser=start, doc=start.format_help())
|
||||
install.set_defaults(command='install', install_parser=install, doc=install.format_help())
|
||||
|
||||
groups = {}
|
||||
for group_name in sorted(interface['groups']):
|
||||
group_parser = sub.add_parser(group_name, group_name=group_name, help=interface['groups'][group_name])
|
||||
groups[group_name] = group_parser.add_subparsers(metavar='COMMAND')
|
||||
|
||||
nicer_order = ['stop', 'get', 'publish', 'resolve']
|
||||
for command_name in sorted(interface['commands']):
|
||||
if command_name not in nicer_order:
|
||||
nicer_order.append(command_name)
|
||||
|
||||
for command_name in nicer_order:
|
||||
command = interface['commands'][command_name]
|
||||
if command.get('group') is None:
|
||||
add_command_parser(sub, command_name, command)
|
||||
else:
|
||||
add_command_parser(groups[command['group']], command_name, command)
|
||||
|
||||
return root
|
||||
|
||||
|
||||
def ensure_directory_exists(path: str):
|
||||
if not os.path.isdir(path):
|
||||
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
async def execute_command(conf, method, params):
|
||||
client = Client(f"http://{conf.api}/ws")
|
||||
await client.connect()
|
||||
responses = await client.send(method, **params)
|
||||
result = await responses.first
|
||||
await client.disconnect()
|
||||
print(result)
|
||||
return result
|
||||
|
||||
|
||||
def normalize_value(x, key=None):
|
||||
if not isinstance(x, str):
|
||||
return x
|
||||
if key in ('uri', 'channel_name', 'name', 'file_name', 'claim_name', 'download_directory'):
|
||||
return x
|
||||
if x.lower() == 'true':
|
||||
return True
|
||||
if x.lower() == 'false':
|
||||
return False
|
||||
if x.isdigit():
|
||||
return int(x)
|
||||
return x
|
||||
|
||||
|
||||
def remove_brackets(key):
|
||||
if key.startswith("<") and key.endswith(">"):
|
||||
return str(key[1:-1])
|
||||
return key
|
||||
|
||||
|
||||
def set_kwargs(parsed_args):
|
||||
kwargs = {}
|
||||
for key, arg in parsed_args.items():
|
||||
if arg is None:
|
||||
continue
|
||||
k = None
|
||||
if key.startswith("--") and remove_brackets(key[2:]) not in kwargs:
|
||||
k = remove_brackets(key[2:])
|
||||
elif remove_brackets(key) not in kwargs:
|
||||
k = remove_brackets(key)
|
||||
kwargs[k] = normalize_value(arg, k)
|
||||
return kwargs
|
||||
|
||||
|
||||
def install_systemd_service():
|
||||
systemd_service = textwrap.dedent(f"""\
|
||||
[Unit]
|
||||
Description=LBRYnet
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart={sys.argv[0]} start --full-node
|
||||
""")
|
||||
subprocess.run(
|
||||
["systemctl", "edit", "--user", "--full", "--force", "lbrynet.service"],
|
||||
input=systemd_service, text=True, check=True,
|
||||
env=dict(os.environ, SYSTEMD_EDITOR="cp /dev/stdin"),
|
||||
)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
conf = Config()
|
||||
conf.set_arguments(args)
|
||||
conf.set_environment()
|
||||
conf.set_default_paths()
|
||||
conf.set_persisted()
|
||||
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
|
||||
ensure_directory_exists(directory)
|
||||
|
||||
if args.cli_version:
|
||||
print(f"lbrynet {__version__}")
|
||||
elif args.command == 'start':
|
||||
if args.help:
|
||||
args.start_parser.print_help()
|
||||
elif args.service == FullNode.name:
|
||||
return Daemon.from_config(FullNode, conf).run()
|
||||
elif args.service == LightClient.name:
|
||||
return Daemon.from_config(LightClient, conf).run()
|
||||
else:
|
||||
print(f'Only `start {FullNode.name}` or `start {LightClient.name}` is currently supported.')
|
||||
elif args.command == 'install':
|
||||
if args.help:
|
||||
args.install_parser.print_help()
|
||||
elif args.system == 'systemd':
|
||||
install_systemd_service()
|
||||
elif args.command is not None:
|
||||
doc = args.doc
|
||||
api_method_name = args.api_method_name
|
||||
if args.help:
|
||||
print(doc)
|
||||
else:
|
||||
parsed = docopt(
|
||||
# TODO: ugly hack because docopt doesn't support commands with spaces in them
|
||||
doc.replace(api_method_name.replace('_', ' '), api_method_name, 1),
|
||||
command_args
|
||||
)
|
||||
params = set_kwargs(parsed)
|
||||
asyncio.get_event_loop().run_until_complete(execute_command(conf, api_method_name, params))
|
||||
elif args.group is not None:
|
||||
args.group_parser.print_help()
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
-759
@@ -1,759 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import typing
|
||||
import logging
|
||||
from argparse import ArgumentParser
|
||||
from contextlib import contextmanager
|
||||
from typing import Tuple
|
||||
|
||||
import yaml
|
||||
from lbry.utils.dirs import user_data_dir, user_download_dir
|
||||
from lbry.error import InvalidCurrencyError
|
||||
from lbry.dht import constants
|
||||
from lbry.wallet.coinselection import COIN_SELECTION_STRATEGIES
|
||||
from lbry.event import EventRegistry
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
NOT_SET = type('NOT_SET', (object,), {}) # pylint: disable=invalid-name
|
||||
T = typing.TypeVar('T')
|
||||
|
||||
CURRENCIES = {
|
||||
'BTC': {'type': 'crypto'},
|
||||
'LBC': {'type': 'crypto'},
|
||||
'USD': {'type': 'fiat'},
|
||||
}
|
||||
|
||||
|
||||
class Setting(typing.Generic[T]):
|
||||
|
||||
def __init__(self, doc: str, default: typing.Optional[T] = None,
|
||||
previous_names: typing.Optional[typing.List[str]] = None,
|
||||
metavar: typing.Optional[str] = None):
|
||||
self.doc = doc
|
||||
self.default = default
|
||||
self.previous_names = previous_names or []
|
||||
self.metavar = metavar
|
||||
|
||||
def __set_name__(self, owner, name):
|
||||
self.name = name # pylint: disable=attribute-defined-outside-init
|
||||
|
||||
@property
|
||||
def cli_name(self):
|
||||
return f"--{self.name.replace('_', '-')}"
|
||||
|
||||
@property
|
||||
def no_cli_name(self):
|
||||
return f"--no-{self.name.replace('_', '-')}"
|
||||
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T:
|
||||
if obj is None:
|
||||
return self
|
||||
for location in obj.search_order:
|
||||
if self.name in location:
|
||||
return location[self.name]
|
||||
return self.default
|
||||
|
||||
def __set__(self, obj: 'BaseConfig', val: typing.Union[T, NOT_SET]):
|
||||
if val == NOT_SET:
|
||||
for location in obj.modify_order:
|
||||
if self.name in location:
|
||||
del location[self.name]
|
||||
else:
|
||||
self.validate(val)
|
||||
for location in obj.modify_order:
|
||||
location[self.name] = val
|
||||
|
||||
def validate(self, value):
|
||||
raise NotImplementedError()
|
||||
|
||||
def deserialize(self, value): # pylint: disable=no-self-use
|
||||
return value
|
||||
|
||||
def serialize(self, value): # pylint: disable=no-self-use
|
||||
return value
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
metavar=self.metavar,
|
||||
default=NOT_SET
|
||||
)
|
||||
|
||||
|
||||
class String(Setting[str]):
|
||||
def validate(self, value):
|
||||
assert isinstance(value, str), \
|
||||
f"Setting '{self.name}' must be a string."
|
||||
|
||||
# TODO: removes this after pylint starts to understand generics
|
||||
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> str: # pylint: disable=useless-super-delegation
|
||||
return super().__get__(obj, owner)
|
||||
|
||||
|
||||
class Integer(Setting[int]):
|
||||
def validate(self, value):
|
||||
assert isinstance(value, int), \
|
||||
f"Setting '{self.name}' must be an integer."
|
||||
|
||||
def deserialize(self, value):
|
||||
return int(value)
|
||||
|
||||
|
||||
class Float(Setting[float]):
|
||||
def validate(self, value):
|
||||
assert isinstance(value, float), \
|
||||
f"Setting '{self.name}' must be a decimal."
|
||||
|
||||
def deserialize(self, value):
|
||||
return float(value)
|
||||
|
||||
|
||||
class Toggle(Setting[bool]):
|
||||
def validate(self, value):
|
||||
assert isinstance(value, bool), \
|
||||
f"Setting '{self.name}' must be a true/false value."
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
action="store_true",
|
||||
default=NOT_SET
|
||||
)
|
||||
parser.add_argument(
|
||||
self.no_cli_name,
|
||||
help=f"Opposite of {self.cli_name}",
|
||||
dest=self.name,
|
||||
action="store_false",
|
||||
default=NOT_SET
|
||||
)
|
||||
|
||||
|
||||
class Path(String):
|
||||
def __init__(self, doc: str, *args, default: str = '', **kwargs):
|
||||
super().__init__(doc, default, *args, **kwargs)
|
||||
|
||||
def __get__(self, obj, owner) -> str:
|
||||
value = super().__get__(obj, owner)
|
||||
if isinstance(value, str):
|
||||
return os.path.expanduser(os.path.expandvars(value))
|
||||
return value
|
||||
|
||||
|
||||
class MaxKeyFee(Setting[dict]):
|
||||
|
||||
def validate(self, value):
|
||||
if value is not None:
|
||||
assert isinstance(value, dict) and set(value) == {'currency', 'amount'}, \
|
||||
f"Setting '{self.name}' must be a dict like \"{{'amount': 50.0, 'currency': 'USD'}}\"."
|
||||
if value["currency"] not in CURRENCIES:
|
||||
raise InvalidCurrencyError(value["currency"])
|
||||
|
||||
@staticmethod
|
||||
def _parse_list(l):
|
||||
if l == ['null']:
|
||||
return None
|
||||
assert len(l) == 2, (
|
||||
'Max key fee is made up of either two values: '
|
||||
'"AMOUNT CURRENCY", or "null" (to set no limit)'
|
||||
)
|
||||
try:
|
||||
amount = float(l[0])
|
||||
except ValueError:
|
||||
raise AssertionError('First value in max key fee is a decimal: "AMOUNT CURRENCY"')
|
||||
currency = str(l[1]).upper()
|
||||
if currency not in CURRENCIES:
|
||||
raise InvalidCurrencyError(currency)
|
||||
return {'amount': amount, 'currency': currency}
|
||||
|
||||
def deserialize(self, value):
|
||||
if value is None:
|
||||
return
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
'currency': value['currency'],
|
||||
'amount': float(value['amount']),
|
||||
}
|
||||
if isinstance(value, str):
|
||||
value = value.split()
|
||||
if isinstance(value, list):
|
||||
return self._parse_list(value)
|
||||
raise AssertionError('Invalid max key fee.')
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
nargs='+',
|
||||
metavar=('AMOUNT', 'CURRENCY'),
|
||||
default=NOT_SET
|
||||
)
|
||||
parser.add_argument(
|
||||
self.no_cli_name,
|
||||
help="Disable maximum key fee check.",
|
||||
dest=self.name,
|
||||
const=None,
|
||||
action="store_const",
|
||||
default=NOT_SET
|
||||
)
|
||||
|
||||
|
||||
class StringChoice(String):
|
||||
def __init__(self, doc: str, valid_values: typing.List[str], default: str, *args, **kwargs):
|
||||
super().__init__(doc, default, *args, **kwargs)
|
||||
if not valid_values:
|
||||
raise ValueError("No valid values provided")
|
||||
if default not in valid_values:
|
||||
raise ValueError(f"Default value must be one of: {', '.join(valid_values)}")
|
||||
self.valid_values = valid_values
|
||||
|
||||
def validate(self, value):
|
||||
super().validate(value)
|
||||
if value not in self.valid_values:
|
||||
raise ValueError(f"Setting '{self.name}' value must be one of: {', '.join(self.valid_values)}")
|
||||
|
||||
|
||||
class ListSetting(Setting[list]):
|
||||
|
||||
def validate(self, value):
|
||||
assert isinstance(value, (tuple, list)), \
|
||||
f"Setting '{self.name}' must be a tuple or list."
|
||||
|
||||
def contribute_to_argparse(self, parser: ArgumentParser):
|
||||
parser.add_argument(
|
||||
self.cli_name,
|
||||
help=self.doc,
|
||||
action='append'
|
||||
)
|
||||
|
||||
|
||||
class Servers(ListSetting):
|
||||
|
||||
def validate(self, value):
|
||||
assert isinstance(value, (tuple, list)), \
|
||||
f"Setting '{self.name}' must be a tuple or list of servers."
|
||||
for idx, server in enumerate(value):
|
||||
assert isinstance(server, (tuple, list)) and len(server) == 2, \
|
||||
f"Server defined '{server}' at index {idx} in setting " \
|
||||
f"'{self.name}' must be a tuple or list of two items."
|
||||
assert isinstance(server[0], str), \
|
||||
f"Server defined '{server}' at index {idx} in setting " \
|
||||
f"'{self.name}' must be have hostname as string in first position."
|
||||
assert isinstance(server[1], int), \
|
||||
f"Server defined '{server}' at index {idx} in setting " \
|
||||
f"'{self.name}' must be have port as int in second position."
|
||||
|
||||
def deserialize(self, value):
|
||||
servers = []
|
||||
if isinstance(value, list):
|
||||
for server in value:
|
||||
if isinstance(server, str) and server.count(':') == 1:
|
||||
host, port = server.split(':')
|
||||
try:
|
||||
servers.append((host, int(port)))
|
||||
except ValueError:
|
||||
pass
|
||||
return servers
|
||||
|
||||
def serialize(self, value):
|
||||
if value:
|
||||
return [f"{host}:{port}" for host, port in value]
|
||||
return value
|
||||
|
||||
|
||||
class Strings(ListSetting):
|
||||
|
||||
def validate(self, value):
|
||||
assert isinstance(value, (tuple, list)), \
|
||||
f"Setting '{self.name}' must be a tuple or list of strings."
|
||||
for idx, string in enumerate(value):
|
||||
assert isinstance(string, str), \
|
||||
f"Value of '{string}' at index {idx} in setting " \
|
||||
f"'{self.name}' must be a string."
|
||||
|
||||
|
||||
class EnvironmentAccess:
|
||||
PREFIX = 'LBRY_'
|
||||
|
||||
def __init__(self, environ: dict):
|
||||
self.environ = environ
|
||||
|
||||
def __contains__(self, item: str):
|
||||
return f'{self.PREFIX}{item.upper()}' in self.environ
|
||||
|
||||
def __getitem__(self, item: str):
|
||||
return self.environ[f'{self.PREFIX}{item.upper()}']
|
||||
|
||||
|
||||
class ArgumentAccess:
|
||||
|
||||
def __init__(self, config: 'BaseConfig', args: dict):
|
||||
self.configuration = config
|
||||
self.args = {}
|
||||
if args:
|
||||
self.load(args)
|
||||
|
||||
def load(self, args):
|
||||
for setting in self.configuration.get_settings():
|
||||
value = getattr(args, setting.name, NOT_SET)
|
||||
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
|
||||
self.args[setting.name] = setting.deserialize(value)
|
||||
|
||||
def __contains__(self, item: str):
|
||||
return item in self.args
|
||||
|
||||
def __getitem__(self, item: str):
|
||||
return self.args[item]
|
||||
|
||||
|
||||
class ConfigFileAccess:
|
||||
|
||||
def __init__(self, config: 'BaseConfig', path: str):
|
||||
self.configuration = config
|
||||
self.path = path
|
||||
self.data = {}
|
||||
if self.exists:
|
||||
self.load()
|
||||
|
||||
@property
|
||||
def exists(self):
|
||||
return self.path and os.path.exists(self.path)
|
||||
|
||||
def load(self):
|
||||
cls = type(self.configuration)
|
||||
with open(self.path, 'r') as config_file:
|
||||
raw = config_file.read()
|
||||
serialized = yaml.full_load(raw) or {}
|
||||
for key, value in serialized.items():
|
||||
attr = getattr(cls, key, None)
|
||||
if attr is None:
|
||||
for setting in self.configuration.settings:
|
||||
if key in setting.previous_names:
|
||||
attr = setting
|
||||
break
|
||||
if attr is not None:
|
||||
self.data[key] = attr.deserialize(value)
|
||||
|
||||
def save(self):
|
||||
cls = type(self.configuration)
|
||||
serialized = {}
|
||||
for key, value in self.data.items():
|
||||
attr = getattr(cls, key)
|
||||
serialized[key] = attr.serialize(value)
|
||||
with open(self.path, 'w') as config_file:
|
||||
config_file.write(yaml.safe_dump(serialized, default_flow_style=False))
|
||||
|
||||
def upgrade(self) -> bool:
|
||||
upgraded = False
|
||||
for key in list(self.data):
|
||||
for setting in self.configuration.settings:
|
||||
if key in setting.previous_names:
|
||||
self.data[setting.name] = self.data[key]
|
||||
del self.data[key]
|
||||
upgraded = True
|
||||
break
|
||||
return upgraded
|
||||
|
||||
def __contains__(self, item: str):
|
||||
return item in self.data
|
||||
|
||||
def __getitem__(self, item: str):
|
||||
return self.data[item]
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self.data[key] = value
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self.data[key]
|
||||
|
||||
|
||||
TBC = typing.TypeVar('TBC', bound='BaseConfig')
|
||||
|
||||
|
||||
class BaseConfig:
|
||||
|
||||
config = Path("Path to configuration file.", metavar='FILE')
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.runtime = {} # set internally or by various API calls
|
||||
self.arguments = {} # from command line arguments
|
||||
self.environment = {} # from environment variables
|
||||
self.persisted = {} # from config file
|
||||
self._updating_config = False
|
||||
self.events = EventRegistry()
|
||||
self.set(**kwargs)
|
||||
|
||||
def set(self, **kwargs):
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
return self
|
||||
|
||||
@contextmanager
|
||||
def update_config(self):
|
||||
self._updating_config = True
|
||||
yield self
|
||||
self._updating_config = False
|
||||
if isinstance(self.persisted, ConfigFileAccess):
|
||||
self.persisted.save()
|
||||
|
||||
@property
|
||||
def modify_order(self):
|
||||
locations = [self.runtime]
|
||||
if self._updating_config:
|
||||
locations.append(self.persisted)
|
||||
return locations
|
||||
|
||||
@property
|
||||
def search_order(self):
|
||||
return [
|
||||
self.runtime,
|
||||
self.arguments,
|
||||
self.environment,
|
||||
self.persisted
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def get_settings(cls):
|
||||
for attr in dir(cls):
|
||||
setting = getattr(cls, attr)
|
||||
if isinstance(setting, Setting):
|
||||
yield setting
|
||||
|
||||
@property
|
||||
def settings(self):
|
||||
return self.get_settings()
|
||||
|
||||
@property
|
||||
def settings_dict(self):
|
||||
return {
|
||||
setting.name: getattr(self, setting.name) for setting in self.settings
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def create_from_arguments(cls, args) -> TBC:
|
||||
conf = cls()
|
||||
conf.set_arguments(args)
|
||||
conf.set_environment()
|
||||
conf.set_persisted()
|
||||
return conf
|
||||
|
||||
@classmethod
|
||||
def contribute_to_argparse(cls, parser: ArgumentParser):
|
||||
for setting in cls.get_settings():
|
||||
setting.contribute_to_argparse(parser)
|
||||
|
||||
def set_arguments(self, args):
|
||||
self.arguments = ArgumentAccess(self, args)
|
||||
|
||||
def set_environment(self, environ=None):
|
||||
self.environment = EnvironmentAccess(environ or os.environ)
|
||||
|
||||
def set_persisted(self, config_file_path=None):
|
||||
if config_file_path is None:
|
||||
config_file_path = self.config
|
||||
|
||||
if not config_file_path:
|
||||
return
|
||||
|
||||
ext = os.path.splitext(config_file_path)[1]
|
||||
assert ext in ('.yml', '.yaml'),\
|
||||
f"File extension '{ext}' is not supported, " \
|
||||
f"configuration file must be in YAML (.yaml)."
|
||||
|
||||
self.persisted = ConfigFileAccess(self, config_file_path)
|
||||
if self.persisted.upgrade():
|
||||
self.persisted.save()
|
||||
|
||||
|
||||
class TranscodeConfig(BaseConfig):
|
||||
|
||||
ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. '
|
||||
f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}',
|
||||
'', previous_names=['ffmpeg_folder'])
|
||||
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
||||
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
||||
'libx264 -crf 21 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000)
|
||||
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
|
||||
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
|
||||
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
|
||||
r'-maxrate 8400K -bufsize 5000K')
|
||||
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
||||
'Example: libopus -b:a 128k',
|
||||
'aac -b:a 160k')
|
||||
volume_filter = String('FFmpeg filter for audio normalization.', '-af loudnorm')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
|
||||
|
||||
|
||||
class CLIConfig(TranscodeConfig):
|
||||
|
||||
api = String('Host name and port for lbrynet daemon API.', 'localhost:5279', metavar='HOST:PORT')
|
||||
|
||||
@property
|
||||
def api_connection_url(self) -> str:
|
||||
return f"http://{self.api}/lbryapi"
|
||||
|
||||
@property
|
||||
def api_host(self):
|
||||
return self.api.split(':')[0]
|
||||
|
||||
@property
|
||||
def api_port(self):
|
||||
return int(self.api.split(':')[1])
|
||||
|
||||
|
||||
class Config(CLIConfig):
|
||||
db_url = String("Database connection URL, uses a local file based SQLite by default.")
|
||||
workers = Integer(
|
||||
"Multiprocessing, specify number of worker processes lbrynet can start (including main process)."
|
||||
" (-1: threads only, 0: equal to number of CPUs, >1: specific number of processes)", -1
|
||||
)
|
||||
console = StringChoice(
|
||||
"Basic text console output or advanced colored output with progress bars.",
|
||||
["basic", "advanced", "none"], "advanced"
|
||||
)
|
||||
|
||||
# directories
|
||||
download_dir = Path("Directory to store downloaded files.", metavar='DIR')
|
||||
data_dir = Path("Main directory containing blobs, wallets and blockchain data.", metavar='DIR')
|
||||
blob_dir = Path("Directory to store blobs (default: 'data_dir'/blobs).", metavar='DIR')
|
||||
wallet_dir = Path("Directory to store wallets (default: 'data_dir'/wallets).", metavar='DIR')
|
||||
wallet_storage = StringChoice("Wallet storage mode.", ["file", "database"], "file")
|
||||
wallets = Strings(
|
||||
"Wallet files in 'wallet_dir' to load at startup.", ['default_wallet']
|
||||
)
|
||||
create_default_wallet = Toggle(
|
||||
"Create an initial wallet if it does not exist on startup.", True
|
||||
)
|
||||
create_default_account = Toggle(
|
||||
"Create an initial account if it does not exist in the default wallet.", True
|
||||
)
|
||||
|
||||
# network
|
||||
use_upnp = Toggle(
|
||||
"Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward"
|
||||
"ports or have firewall rules you likely want to disable this.", True
|
||||
)
|
||||
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
|
||||
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
|
||||
prometheus_port = Integer("Port to expose prometheus metrics (off by default)", 0)
|
||||
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
|
||||
|
||||
# routing table
|
||||
split_buckets_under_index = Integer(
|
||||
"Routing table bucket index below which we always split the bucket if given a new key to add to it and "
|
||||
"the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
|
||||
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
|
||||
"use.", 1
|
||||
)
|
||||
|
||||
# protocol timeouts
|
||||
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
|
||||
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
|
||||
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
|
||||
node_rpc_timeout = Float("Timeout when making a DHT request", constants.RPC_TIMEOUT)
|
||||
|
||||
# blob announcement and download
|
||||
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
|
||||
blob_lru_cache_size = Integer(
|
||||
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
|
||||
"replying to a range request. Set to 0 to disable.", 32
|
||||
)
|
||||
announce_head_and_sd_only = Toggle(
|
||||
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
|
||||
previous_names=['announce_head_blobs_only']
|
||||
)
|
||||
concurrent_blob_announcers = Integer(
|
||||
"Number of blobs to iteratively announce at once, set to 0 to disable", 10,
|
||||
previous_names=['concurrent_announcers']
|
||||
)
|
||||
max_connections_per_download = Integer(
|
||||
"Maximum number of peers to connect to while downloading a blob", 4,
|
||||
previous_names=['max_connections_per_stream']
|
||||
)
|
||||
fixed_peer_delay = Float(
|
||||
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
|
||||
"peers are not found or are slow", 2.0
|
||||
)
|
||||
max_key_fee = MaxKeyFee(
|
||||
"Don't download streams with fees exceeding this amount. When set to "
|
||||
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
|
||||
)
|
||||
max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0")
|
||||
|
||||
# reflector settings
|
||||
reflect_streams = Toggle(
|
||||
"Upload completed streams (published and downloaded) reflector in order to re-host them", True,
|
||||
previous_names=['reflect_uploads']
|
||||
)
|
||||
concurrent_reflector_uploads = Integer(
|
||||
"Maximum number of streams to upload to a reflector server at a time", 10
|
||||
)
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
('reflector.lbry.com', 5566)
|
||||
])
|
||||
full_nodes = Servers("Full blockchain nodes", [
|
||||
('sdk.lbry.tech', 5278),
|
||||
('spv12.lbry.com', 50001),
|
||||
('spv13.lbry.com', 50001),
|
||||
('spv14.lbry.com', 50001),
|
||||
('spv15.lbry.com', 50001),
|
||||
('spv16.lbry.com', 50001),
|
||||
('spv17.lbry.com', 50001),
|
||||
('spv18.lbry.com', 50001),
|
||||
('spv19.lbry.com', 50001),
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
('lbrynet2.lbry.com', 4444), # US WEST
|
||||
('lbrynet3.lbry.com', 4444), # EU
|
||||
('lbrynet4.lbry.com', 4444) # ASIA
|
||||
])
|
||||
|
||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
|
||||
|
||||
# blockchain
|
||||
blockchain = StringChoice("Blockchain network type.", ["mainnet", "regtest", "testnet"], "mainnet")
|
||||
lbrycrd_rpc_user = String("Username for connecting to lbrycrd.", "rpcuser")
|
||||
lbrycrd_rpc_pass = String("Password for connecting to lbrycrd.", "rpcpassword")
|
||||
lbrycrd_rpc_host = String("Hostname for connecting to lbrycrd.", "localhost")
|
||||
lbrycrd_rpc_port = Integer("Port for connecting to lbrycrd.", 9245)
|
||||
lbrycrd_peer_port = Integer("Peer port for lbrycrd.", 9246)
|
||||
lbrycrd_zmq = String("ZMQ events address.")
|
||||
lbrycrd_dir = Path("Directory containing lbrycrd data.", metavar='DIR')
|
||||
search_censor_channel_ids = Strings("List of channel ids for filtering out search results.", [])
|
||||
resolve_censor_channel_ids = Strings("List of channel ids for filtering out resolve results.", [])
|
||||
|
||||
# daemon
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", False,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
track_bandwidth = Toggle("Track bandwidth usage", True)
|
||||
|
||||
# media server
|
||||
streaming_server = String('Host name and port to serve streaming media over range requests',
|
||||
'localhost:5280', metavar='HOST:PORT')
|
||||
streaming_get = Toggle("Enable the /get endpoint for the streaming media server. "
|
||||
"Disable to prevent new streams from being added.", True)
|
||||
|
||||
coin_selection_strategy = StringChoice(
|
||||
"Strategy to use when selecting UTXOs for a transaction",
|
||||
COIN_SELECTION_STRATEGIES, "standard")
|
||||
|
||||
save_resolved_claims = Toggle(
|
||||
"Save content claims to the database when they are resolved to keep file_list up to date, "
|
||||
"only disable this if file_x commands are not needed", True
|
||||
)
|
||||
|
||||
@property
|
||||
def streaming_host(self):
|
||||
return self.streaming_server.split(':')[0]
|
||||
|
||||
@property
|
||||
def streaming_port(self):
|
||||
return int(self.streaming_server.split(':')[1])
|
||||
|
||||
@classmethod
|
||||
def with_null_dir(cls):
|
||||
return cls.with_same_dir('/dev/null')
|
||||
|
||||
@classmethod
|
||||
def with_same_dir(cls, same_dir):
|
||||
return cls(
|
||||
data_dir=same_dir,
|
||||
download_dir=same_dir,
|
||||
wallet_dir=same_dir,
|
||||
lbrycrd_dir=same_dir,
|
||||
)
|
||||
|
||||
def set_default_paths(self):
|
||||
if 'darwin' in sys.platform.lower():
|
||||
get_directories = get_darwin_directories
|
||||
elif 'win' in sys.platform.lower():
|
||||
get_directories = get_windows_directories
|
||||
elif 'linux' in sys.platform.lower():
|
||||
get_directories = get_linux_directories
|
||||
else:
|
||||
return
|
||||
cls = type(self)
|
||||
cls.data_dir.default, cls.wallet_dir.default,\
|
||||
cls.blob_dir.default, cls.download_dir.default = get_directories()
|
||||
old_settings_file = os.path.join(self.data_dir, 'daemon_settings.yml')
|
||||
if os.path.exists(old_settings_file):
|
||||
cls.config.default = old_settings_file
|
||||
else:
|
||||
cls.config.default = os.path.join(self.data_dir, 'settings.yml')
|
||||
if self.data_dir != cls.data_dir.default:
|
||||
cls.blob_dir.default = os.path.join(self.data_dir, 'blobs')
|
||||
cls.wallet_dir.default = os.path.join(self.data_dir, 'wallets')
|
||||
|
||||
@property
|
||||
def log_file_path(self):
|
||||
return os.path.join(self.data_dir, 'daemon.log')
|
||||
|
||||
@property
|
||||
def db_url_or_default(self):
|
||||
if self.db_url:
|
||||
return self.db_url
|
||||
return 'sqlite:///'+os.path.join(self.data_dir, f'{self.blockchain}.db')
|
||||
|
||||
|
||||
def get_windows_directories() -> Tuple[str, str, str, str]:
|
||||
# very old
|
||||
data_dir = user_data_dir('lbrynet', roaming=True)
|
||||
blob_dir = os.path.join(data_dir, 'blobfiles')
|
||||
wallet_dir = os.path.join(user_data_dir('lbryum', roaming=True), 'wallets')
|
||||
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
|
||||
return data_dir, wallet_dir, blob_dir, user_download_dir()
|
||||
# old
|
||||
data_dir = user_data_dir('lbrynet', 'lbry')
|
||||
blob_dir = os.path.join(data_dir, 'blobfiles')
|
||||
wallet_dir = os.path.join(user_data_dir('lbryum', 'lbry'), 'wallets')
|
||||
if os.path.isdir(blob_dir) and os.path.isdir(wallet_dir):
|
||||
return data_dir, wallet_dir, blob_dir, user_download_dir()
|
||||
# new
|
||||
return get_universal_directories()
|
||||
|
||||
|
||||
def get_darwin_directories() -> Tuple[str, str, str, str]:
|
||||
data_dir = user_data_dir('LBRY')
|
||||
blob_dir = os.path.join(data_dir, 'blobfiles')
|
||||
wallet_dir = os.path.expanduser('~/.lbryum/wallets')
|
||||
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
|
||||
return data_dir, wallet_dir, blob_dir, user_download_dir()
|
||||
return get_universal_directories()
|
||||
|
||||
|
||||
def get_linux_directories() -> Tuple[str, str, str, str]:
|
||||
# very old
|
||||
data_dir = os.path.expanduser('~/.lbrynet')
|
||||
blob_dir = os.path.join(data_dir, 'blobfiles')
|
||||
wallet_dir = os.path.join(os.path.expanduser('~/.lbryum'), 'wallets')
|
||||
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
|
||||
return data_dir, wallet_dir, blob_dir, user_download_dir()
|
||||
# old
|
||||
data_dir = user_data_dir('lbry/lbrynet')
|
||||
blob_dir = os.path.join(data_dir, 'blobfiles')
|
||||
wallet_dir = user_data_dir('lbry/lbryum/wallets')
|
||||
if os.path.isdir(blob_dir) or os.path.isdir(wallet_dir):
|
||||
return data_dir, wallet_dir, blob_dir, user_download_dir()
|
||||
# new
|
||||
return get_universal_directories()
|
||||
|
||||
|
||||
def get_universal_directories() -> Tuple[str, str, str, str]:
|
||||
lbrynet_dir = user_data_dir('lbrynet', 'LBRY')
|
||||
return (
|
||||
lbrynet_dir,
|
||||
os.path.join(lbrynet_dir, 'wallets'),
|
||||
os.path.join(lbrynet_dir, 'blobs'),
|
||||
user_download_dir()
|
||||
)
|
||||
@@ -1,105 +0,0 @@
|
||||
import time
|
||||
import asyncio
|
||||
import typing
|
||||
import collections
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
CONNECTED_EVENT = "connected"
|
||||
DISCONNECTED_EVENT = "disconnected"
|
||||
TRANSFERRED_EVENT = "transferred"
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop):
|
||||
self.loop = loop
|
||||
self.incoming_connected: typing.Set[str] = set()
|
||||
self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int)
|
||||
self.outgoing_connected: typing.Set[str] = set()
|
||||
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
|
||||
self._max_incoming_mbs = 0.0
|
||||
self._max_outgoing_mbs = 0.0
|
||||
self._status = {}
|
||||
self._running = False
|
||||
self._task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
@property
|
||||
def status(self):
|
||||
return self._status
|
||||
|
||||
def sent_data(self, host_and_port: str, size: int):
|
||||
if self._running:
|
||||
self.outgoing[host_and_port] += size
|
||||
|
||||
def received_data(self, host_and_port: str, size: int):
|
||||
if self._running:
|
||||
self.incoming[host_and_port] += size
|
||||
|
||||
def connection_made(self, host_and_port: str):
|
||||
if self._running:
|
||||
self.outgoing_connected.add(host_and_port)
|
||||
|
||||
def connection_received(self, host_and_port: str):
|
||||
# self.incoming_connected.add(host_and_port)
|
||||
pass
|
||||
|
||||
def outgoing_connection_lost(self, host_and_port: str):
|
||||
if self._running and host_and_port in self.outgoing_connected:
|
||||
self.outgoing_connected.remove(host_and_port)
|
||||
|
||||
def incoming_connection_lost(self, host_and_port: str):
|
||||
if self._running and host_and_port in self.incoming_connected:
|
||||
self.incoming_connected.remove(host_and_port)
|
||||
|
||||
async def _update(self):
|
||||
self._status = {
|
||||
'incoming_bps': {},
|
||||
'outgoing_bps': {},
|
||||
'total_incoming_mbs': 0.0,
|
||||
'total_outgoing_mbs': 0.0,
|
||||
'total_sent': 0,
|
||||
'total_received': 0,
|
||||
'max_incoming_mbs': 0.0,
|
||||
'max_outgoing_mbs': 0.0
|
||||
}
|
||||
|
||||
while True:
|
||||
last = time.perf_counter()
|
||||
await asyncio.sleep(0.1, loop=self.loop)
|
||||
self._status['incoming_bps'].clear()
|
||||
self._status['outgoing_bps'].clear()
|
||||
now = time.perf_counter()
|
||||
while self.outgoing:
|
||||
k, sent = self.outgoing.popitem()
|
||||
self._status['total_sent'] += sent
|
||||
self._status['outgoing_bps'][k] = sent / (now - last)
|
||||
while self.incoming:
|
||||
k, received = self.incoming.popitem()
|
||||
self._status['total_received'] += received
|
||||
self._status['incoming_bps'][k] = received / (now - last)
|
||||
self._status['total_outgoing_mbs'] = int(sum(list(self._status['outgoing_bps'].values())
|
||||
)) / 1000000.0
|
||||
self._status['total_incoming_mbs'] = int(sum(list(self._status['incoming_bps'].values())
|
||||
)) / 1000000.0
|
||||
self._max_incoming_mbs = max(self._max_incoming_mbs, self._status['total_incoming_mbs'])
|
||||
self._max_outgoing_mbs = max(self._max_outgoing_mbs, self._status['total_outgoing_mbs'])
|
||||
self._status['max_incoming_mbs'] = self._max_incoming_mbs
|
||||
self._status['max_outgoing_mbs'] = self._max_outgoing_mbs
|
||||
|
||||
def stop(self):
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
self.outgoing.clear()
|
||||
self.outgoing_connected.clear()
|
||||
self.incoming.clear()
|
||||
self.incoming_connected.clear()
|
||||
self._status.clear()
|
||||
self._running = False
|
||||
|
||||
def start(self):
|
||||
self.stop()
|
||||
self._running = True
|
||||
self._task = self.loop.create_task(self._update())
|
||||
-496
@@ -1,496 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import itertools
|
||||
import logging
|
||||
from typing import Dict, Any, Type
|
||||
from tempfile import TemporaryFile
|
||||
|
||||
from tqdm.std import tqdm, Bar
|
||||
from tqdm.utils import FormatReplace, _unicode, disp_len, disp_trim, _is_ascii
|
||||
|
||||
from lbry import __version__
|
||||
from lbry.service.base import Service
|
||||
from lbry.service.full_node import FullNode
|
||||
from lbry.service.light_client import LightClient
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RedirectOutput:
|
||||
|
||||
silence_lines = [
|
||||
b'libprotobuf ERROR google/protobuf/wire_format_lite.cc:626',
|
||||
]
|
||||
|
||||
def __init__(self, stream_type: str):
|
||||
assert stream_type in ('stderr', 'stdout')
|
||||
self.stream_type = stream_type
|
||||
self.stream_no = getattr(sys, stream_type).fileno()
|
||||
self.last_flush = time.time()
|
||||
self.last_read = 0
|
||||
self.backup = None
|
||||
self.file = None
|
||||
|
||||
def __enter__(self):
|
||||
self.backup = os.dup(self.stream_no)
|
||||
setattr(sys, self.stream_type, os.fdopen(self.backup, 'w'))
|
||||
self.file = TemporaryFile()
|
||||
self.backup = os.dup2(self.file.fileno(), self.stream_no)
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.file.close()
|
||||
os.dup2(self.backup, self.stream_no)
|
||||
os.close(self.backup)
|
||||
setattr(sys, self.stream_type, os.fdopen(self.stream_no, 'w'))
|
||||
|
||||
def capture(self):
|
||||
self.__enter__()
|
||||
|
||||
def release(self):
|
||||
self.__exit__(None, None, None)
|
||||
|
||||
def flush(self, writer, force=False):
|
||||
if not force and (time.time() - self.last_flush) < 5:
|
||||
return
|
||||
self.file.seek(self.last_read)
|
||||
for line in self.file.readlines():
|
||||
silence = False
|
||||
for bad_line in self.silence_lines:
|
||||
if bad_line in line:
|
||||
silence = True
|
||||
break
|
||||
if not silence:
|
||||
writer(line.decode().rstrip())
|
||||
self.last_read = self.file.tell()
|
||||
self.last_flush = time.time()
|
||||
|
||||
|
||||
class Console:
|
||||
|
||||
def __init__(self, service: Service):
|
||||
self.service = service
|
||||
|
||||
def starting(self):
|
||||
pass
|
||||
|
||||
def stopping(self):
|
||||
pass
|
||||
|
||||
|
||||
class Basic(Console):
|
||||
|
||||
def __init__(self, service: Service):
|
||||
super().__init__(service)
|
||||
self.service.sync.on_progress.listen(self.on_sync_progress)
|
||||
self.tasks = {}
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
|
||||
|
||||
def starting(self):
|
||||
conf = self.service.conf
|
||||
s = [f'LBRY v{__version__}']
|
||||
if isinstance(self.service, FullNode):
|
||||
s.append('Full Node')
|
||||
elif isinstance(self.service, LightClient):
|
||||
s.append('Light Client')
|
||||
if conf.workers == -1:
|
||||
s.append('Threads Only')
|
||||
else:
|
||||
workers = os.cpu_count() if conf.workers == 0 else conf.workers
|
||||
s.append(f'{workers} Worker' if workers == 1 else f'{workers} Workers')
|
||||
s.append(f'({os.cpu_count()} CPUs available)')
|
||||
log.info(' '.join(s))
|
||||
|
||||
def stopping(self):
|
||||
log.info('exiting')
|
||||
|
||||
@staticmethod
|
||||
def maybe_log_progress(event, done, total, last):
|
||||
if done == 0:
|
||||
log.info("%s 0%%", event)
|
||||
return 0
|
||||
elif done == total:
|
||||
log.info("%s 100%%", event)
|
||||
return 1
|
||||
else:
|
||||
percent = done/total
|
||||
if percent >= 0.25 > last:
|
||||
log.info("%s 25%%", event)
|
||||
return 0.25
|
||||
elif percent >= 0.50 > last:
|
||||
log.info("%s 50%%", event)
|
||||
return 0.50
|
||||
elif percent >= 0.75 > last:
|
||||
log.info("%s 75%%", event)
|
||||
return 0.75
|
||||
return last
|
||||
|
||||
def on_sync_progress(self, event):
|
||||
e, data = event["event"], event["data"]
|
||||
name, current, total, last = e, data['done'][0], 0, 0
|
||||
if not e.endswith("init") and not e.endswith("main") and not e.endswith("indexes"):
|
||||
name = f"{e}#{data['id']}"
|
||||
if "total" in data:
|
||||
total, last = self.tasks[name] = (data["total"][0], last)
|
||||
elif name in self.tasks:
|
||||
total, last = self.tasks[name]
|
||||
elif total == 0:
|
||||
return
|
||||
progress_status = (total, self.maybe_log_progress(name, current, total, last))
|
||||
if progress_status[1] == 1:
|
||||
del self.tasks[name]
|
||||
else:
|
||||
self.tasks[name] = progress_status
|
||||
|
||||
|
||||
class Bar2(Bar):
|
||||
|
||||
def __init__(self, frac, default_len=10, charset=None):
|
||||
super().__init__(frac[0], default_len, charset)
|
||||
self.frac2 = frac[1]
|
||||
|
||||
def __format__(self, format_spec):
|
||||
width = self.default_len
|
||||
row1 = (1,)*int(self.frac2 * width * 2)
|
||||
row2 = (2,)*int(self.frac * width * 2)
|
||||
fill = []
|
||||
for one, two, _ in itertools.zip_longest(row1, row2, range(width*2)):
|
||||
fill.append((one or 0)+(two or 0))
|
||||
bar = []
|
||||
for i in range(0, width*2, 2):
|
||||
if fill[i] == 1:
|
||||
if fill[i+1] == 1:
|
||||
bar.append('▀')
|
||||
else:
|
||||
bar.append('▘')
|
||||
elif fill[i] == 2:
|
||||
if fill[i+1] == 2:
|
||||
bar.append('▄')
|
||||
else:
|
||||
bar.append('▖')
|
||||
elif fill[i] == 3:
|
||||
if fill[i+1] == 1:
|
||||
bar.append('▛')
|
||||
elif fill[i+1] == 2:
|
||||
bar.append('▙')
|
||||
elif fill[i+1] == 3:
|
||||
bar.append('█')
|
||||
else:
|
||||
bar.append('▌')
|
||||
else:
|
||||
bar.append(' ')
|
||||
return ''.join(bar)
|
||||
|
||||
|
||||
class tqdm2(tqdm): # pylint: disable=invalid-name
|
||||
|
||||
def __init__(self, initial=(0, 0), unit=('it', 'it'), total=(None, None), **kwargs):
|
||||
self.n2 = self.last_print_n2 = initial[1] # pylint: disable=invalid-name
|
||||
self.unit2 = unit[1]
|
||||
self.total2 = total[1]
|
||||
super().__init__(initial=initial[0], unit=unit[0], total=total[0], **kwargs)
|
||||
|
||||
@property
|
||||
def format_dict(self):
|
||||
d = super().format_dict
|
||||
d.update({
|
||||
'n2': self.n2,
|
||||
'unit2': self.unit2,
|
||||
'total2': self.total2,
|
||||
})
|
||||
return d
|
||||
|
||||
def update(self, n=(1, 1)):
|
||||
if self.disable:
|
||||
return
|
||||
last_last_print_t = self.last_print_t
|
||||
self.n2 += n[1]
|
||||
super().update(n[0])
|
||||
if last_last_print_t != self.last_print_t:
|
||||
self.last_print_n2 = self.n2
|
||||
|
||||
@staticmethod
|
||||
def format_meter(
|
||||
n, total, elapsed, ncols=None, prefix='', ascii=False, # pylint: disable=redefined-builtin
|
||||
unit='it', unit_scale=False, rate=None, bar_format=None,
|
||||
postfix=None, unit_divisor=1000, initial=0, **extra_kwargs
|
||||
):
|
||||
|
||||
# sanity check: total
|
||||
if total and n >= (total + 0.5): # allow float imprecision (#849)
|
||||
total = None
|
||||
|
||||
# apply custom scale if necessary
|
||||
if unit_scale and unit_scale not in (True, 1):
|
||||
if total:
|
||||
total *= unit_scale
|
||||
n *= unit_scale
|
||||
if rate:
|
||||
rate *= unit_scale # by default rate = 1 / self.avg_time
|
||||
unit_scale = False
|
||||
|
||||
elapsed_str = tqdm.format_interval(elapsed)
|
||||
|
||||
# if unspecified, attempt to use rate = average speed
|
||||
# (we allow manual override since predicting time is an arcane art)
|
||||
if rate is None and elapsed:
|
||||
rate = n / elapsed
|
||||
inv_rate = 1 / rate if rate else None
|
||||
format_sizeof = tqdm.format_sizeof
|
||||
rate_noinv_fmt = ((format_sizeof(rate) if unit_scale else
|
||||
'{0:5.2f}'.format(rate))
|
||||
if rate else '?') + unit + '/s'
|
||||
rate_inv_fmt = ((format_sizeof(inv_rate) if unit_scale else
|
||||
'{0:5.2f}'.format(inv_rate))
|
||||
if inv_rate else '?') + 's/' + unit
|
||||
rate_fmt = rate_inv_fmt if inv_rate and inv_rate > 1 else rate_noinv_fmt
|
||||
|
||||
if unit_scale:
|
||||
n_fmt = format_sizeof(n, divisor=unit_divisor)
|
||||
total_fmt = format_sizeof(total, divisor=unit_divisor) \
|
||||
if total is not None else '?'
|
||||
else:
|
||||
n_fmt = str(n)
|
||||
total_fmt = str(total) if total is not None else '?'
|
||||
|
||||
try:
|
||||
postfix = ', ' + postfix if postfix else ''
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
remaining = (total - n) / rate if rate and total else 0
|
||||
remaining_str = tqdm.format_interval(remaining) if rate else '?'
|
||||
|
||||
# format the stats displayed to the left and right sides of the bar
|
||||
if prefix:
|
||||
# old prefix setup work around
|
||||
bool_prefix_colon_already = (prefix[-2:] == ": ")
|
||||
l_bar = prefix if bool_prefix_colon_already else prefix + ": "
|
||||
else:
|
||||
l_bar = ''
|
||||
|
||||
r_bar = '| {0}/{1} [{2}<{3}, {4}{5}]'.format(
|
||||
n_fmt, total_fmt, elapsed_str, remaining_str, rate_fmt, postfix)
|
||||
|
||||
# Custom bar formatting
|
||||
# Populate a dict with all available progress indicators
|
||||
format_dict = dict(
|
||||
# slight extension of self.format_dict
|
||||
n=n, n_fmt=n_fmt, total=total, total_fmt=total_fmt,
|
||||
elapsed=elapsed_str, elapsed_s=elapsed,
|
||||
ncols=ncols, desc=prefix or '', unit=unit,
|
||||
rate=inv_rate if inv_rate and inv_rate > 1 else rate,
|
||||
rate_fmt=rate_fmt, rate_noinv=rate,
|
||||
rate_noinv_fmt=rate_noinv_fmt, rate_inv=inv_rate,
|
||||
rate_inv_fmt=rate_inv_fmt,
|
||||
postfix=postfix, unit_divisor=unit_divisor,
|
||||
# plus more useful definitions
|
||||
remaining=remaining_str, remaining_s=remaining,
|
||||
l_bar=l_bar, r_bar=r_bar,
|
||||
**extra_kwargs)
|
||||
|
||||
# total is known: we can predict some stats
|
||||
if total:
|
||||
n2, total2 = extra_kwargs['n2'], extra_kwargs['total2'] # pylint: disable=invalid-name
|
||||
|
||||
# fractional and percentage progress
|
||||
frac = n / total
|
||||
frac2 = n2 / total2
|
||||
percentage = frac * 100
|
||||
|
||||
l_bar += '{0:3.0f}%|'.format(percentage)
|
||||
|
||||
if ncols == 0:
|
||||
return l_bar[:-1] + r_bar[1:]
|
||||
|
||||
format_dict.update(l_bar=l_bar)
|
||||
if bar_format:
|
||||
format_dict.update(percentage=percentage)
|
||||
|
||||
# auto-remove colon for empty `desc`
|
||||
if not prefix:
|
||||
bar_format = bar_format.replace("{desc}: ", '')
|
||||
else:
|
||||
bar_format = "{l_bar}{bar}{r_bar}"
|
||||
|
||||
full_bar = FormatReplace()
|
||||
try:
|
||||
nobar = bar_format.format(bar=full_bar, **format_dict)
|
||||
except UnicodeEncodeError:
|
||||
bar_format = _unicode(bar_format)
|
||||
nobar = bar_format.format(bar=full_bar, **format_dict)
|
||||
if not full_bar.format_called:
|
||||
# no {bar}, we can just format and return
|
||||
return nobar
|
||||
|
||||
# Formatting progress bar space available for bar's display
|
||||
full_bar = Bar2(
|
||||
(frac, frac2),
|
||||
max(1, ncols - disp_len(nobar))
|
||||
if ncols else 10,
|
||||
charset=Bar2.ASCII if ascii is True else ascii or Bar2.UTF)
|
||||
if not _is_ascii(full_bar.charset) and _is_ascii(bar_format):
|
||||
bar_format = _unicode(bar_format)
|
||||
res = bar_format.format(bar=full_bar, **format_dict)
|
||||
return disp_trim(res, ncols) if ncols else res
|
||||
|
||||
elif bar_format:
|
||||
# user-specified bar_format but no total
|
||||
l_bar += '|'
|
||||
format_dict.update(l_bar=l_bar, percentage=0)
|
||||
full_bar = FormatReplace()
|
||||
nobar = bar_format.format(bar=full_bar, **format_dict)
|
||||
if not full_bar.format_called:
|
||||
return nobar
|
||||
full_bar = Bar2(
|
||||
(0, 0),
|
||||
max(1, ncols - disp_len(nobar))
|
||||
if ncols else 10,
|
||||
charset=Bar2.BLANK)
|
||||
res = bar_format.format(bar=full_bar, **format_dict)
|
||||
return disp_trim(res, ncols) if ncols else res
|
||||
else:
|
||||
# no total: no progressbar, ETA, just progress stats
|
||||
return ((prefix + ": ") if prefix else '') + \
|
||||
'{0}{1} [{2}, {3}{4}]'.format(
|
||||
n_fmt, unit, elapsed_str, rate_fmt, postfix)
|
||||
|
||||
|
||||
class Advanced(Basic):
|
||||
|
||||
FORMAT = '{l_bar}{bar}| {n_fmt:>8}/{total_fmt:>8} [{elapsed:>7}<{remaining:>8}, {rate_fmt:>17}]'
|
||||
|
||||
def __init__(self, service: Service):
|
||||
super().__init__(service)
|
||||
self.bars: Dict[Any, tqdm] = {}
|
||||
self.stderr = RedirectOutput('stderr')
|
||||
|
||||
def starting(self):
|
||||
self.stderr.capture()
|
||||
super().starting()
|
||||
|
||||
def stopping(self):
|
||||
for bar in self.bars.values():
|
||||
bar.close()
|
||||
super().stopping()
|
||||
#self.stderr.flush(self.bars['read'].write, True)
|
||||
#self.stderr.release()
|
||||
|
||||
def get_or_create_bar(self, name, desc, units, totals, leave=False, bar_format=None, postfix=None, position=None):
|
||||
bar = self.bars.get(name)
|
||||
if bar is None:
|
||||
if len(units) == 2:
|
||||
bar = self.bars[name] = tqdm2(
|
||||
desc=desc, unit=units, total=totals,
|
||||
bar_format=bar_format or self.FORMAT, leave=leave,
|
||||
postfix=postfix, position=position
|
||||
)
|
||||
else:
|
||||
bar = self.bars[name] = tqdm(
|
||||
desc=desc, unit=units[0], total=totals[0],
|
||||
bar_format=bar_format or self.FORMAT, leave=leave,
|
||||
postfix=postfix, position=position
|
||||
)
|
||||
return bar
|
||||
|
||||
def sync_init(self, name, d):
|
||||
bar_name = f"{name}#{d['id']}"
|
||||
bar = self.bars.get(bar_name)
|
||||
if bar is None:
|
||||
label = d.get('label', name[-11:])
|
||||
self.get_or_create_bar(bar_name, label, d['units'], d['total'], True)
|
||||
else:
|
||||
if d['done'][0] != -1:
|
||||
bar.update(d['done'][0] - bar.n)
|
||||
if d['done'][0] == -1 or d['done'][0] == bar.total:
|
||||
bar.close()
|
||||
self.bars.pop(bar_name)
|
||||
|
||||
def sync_main(self, name, d):
|
||||
bar = self.bars.get(name)
|
||||
if bar is None:
|
||||
label = d.get('label', name[-11:])
|
||||
self.get_or_create_bar(name, label, d['units'], d['total'], True)
|
||||
#self.last_stats = f"{d['txs']:,d} txs, {d['claims']:,d} claims and {d['supports']:,d} supports"
|
||||
#self.get_or_create_bar("read", "├─ blocks read", "blocks", d['blocks'], True)
|
||||
#self.get_or_create_bar("save", "└─┬ txs saved", "txs", d['txs'], True)
|
||||
else:
|
||||
if d['done'] == (-1,)*len(d['done']):
|
||||
base_name = name[:name.rindex('.')]
|
||||
for child_name, child_bar in self.bars.items():
|
||||
if child_name.startswith(base_name):
|
||||
child_bar.close()
|
||||
bar.close()
|
||||
self.bars.pop(name)
|
||||
else:
|
||||
if len(d['done']) == 2:
|
||||
bar.update((d['done'][0]-bar.n, d['done'][1]-bar.n2))
|
||||
else:
|
||||
bar.update(d['done'][0]-bar.n)
|
||||
|
||||
def sync_task(self, name, d):
|
||||
bar_name = f"{name}#{d['id']}"
|
||||
bar = self.bars.get(bar_name)
|
||||
if bar is None:
|
||||
#assert d['done'][0] == 0
|
||||
label = d.get('label', name[-11:])
|
||||
self.get_or_create_bar(
|
||||
f"{name}#{d['id']}", label, d['units'], d['total'],
|
||||
name.split('.')[-1] not in ('insert', 'update', 'file')
|
||||
)
|
||||
else:
|
||||
if d['done'][0] != -1:
|
||||
main_bar_name = f"{name[:name.rindex('.')]}.main"
|
||||
if len(d['done']) > 1:
|
||||
diff = tuple(a-b for a, b in zip(d['done'], (bar.n, bar.n2)))
|
||||
else:
|
||||
diff = d['done'][0] - bar.n
|
||||
if main_bar_name != name:
|
||||
main_bar = self.bars.get(main_bar_name)
|
||||
if main_bar and main_bar.unit == bar.unit:
|
||||
main_bar.update(diff)
|
||||
bar.update(diff)
|
||||
if d['done'][0] == -1 or d['done'][0] == bar.total:
|
||||
bar.close()
|
||||
self.bars.pop(bar_name)
|
||||
|
||||
def update_other_bars(self, e, d):
|
||||
if d['total'] == 0:
|
||||
return
|
||||
bar = self.bars.get(e)
|
||||
if not bar:
|
||||
name = (
|
||||
' '.join(e.split('.')[-2:])
|
||||
.replace('support', 'suprt')
|
||||
.replace('channels', 'chanls')
|
||||
.replace('signatures', 'sigs')
|
||||
)
|
||||
bar = self.get_or_create_bar(e, f"├─ {name:>12}", d['unit'], d['total'], True)
|
||||
diff = d['step']-bar.n
|
||||
bar.update(diff)
|
||||
#if d['step'] == d['total']:
|
||||
#bar.close()
|
||||
|
||||
def on_sync_progress(self, event):
|
||||
e, d = event['event'], event.get('data', {})
|
||||
if e.endswith(".init"):
|
||||
self.sync_init(e, d)
|
||||
elif e.endswith(".main"):
|
||||
self.sync_main(e, d)
|
||||
else:
|
||||
self.sync_task(e, d)
|
||||
|
||||
# if e.endswith("sync.start"):
|
||||
# self.sync_start(d)
|
||||
# self.stderr.flush(self.bars['read'].write)
|
||||
# elif e.endswith("sync.complete"):
|
||||
# self.stderr.flush(self.bars['read'].write, True)
|
||||
# self.sync_complete()
|
||||
# else:
|
||||
# self.stderr.flush(self.bars['read'].write)
|
||||
# self.update_progress(e, d)
|
||||
|
||||
|
||||
def console_class_from_name(name) -> Type[Console]:
|
||||
return {'basic': Basic, 'advanced': Advanced}.get(name, Console)
|
||||
@@ -1,8 +0,0 @@
|
||||
DEFAULT_PAGE_SIZE = 20
|
||||
|
||||
NULL_HASH32 = b'\x00'*32
|
||||
|
||||
CENT = 1000000
|
||||
COIN = 100*CENT
|
||||
|
||||
INVALIDATED_SIGNATURE_GRACE_PERIOD = 50
|
||||
@@ -1,86 +0,0 @@
|
||||
from lbry.crypto.hash import double_sha256
|
||||
from lbry.crypto.util import bytes_to_int, int_to_bytes
|
||||
|
||||
|
||||
class Base58Error(Exception):
|
||||
""" Exception used for Base58 errors. """
|
||||
|
||||
|
||||
class Base58:
|
||||
""" Class providing base 58 functionality. """
|
||||
|
||||
chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
|
||||
assert len(chars) == 58
|
||||
char_map = {c: n for n, c in enumerate(chars)}
|
||||
|
||||
@classmethod
|
||||
def char_value(cls, c):
|
||||
val = cls.char_map.get(c)
|
||||
if val is None:
|
||||
raise Base58Error(f'invalid base 58 character "{c}"')
|
||||
return val
|
||||
|
||||
@classmethod
|
||||
def decode(cls, txt):
|
||||
""" Decodes txt into a big-endian bytearray. """
|
||||
if isinstance(txt, memoryview):
|
||||
txt = str(txt)
|
||||
|
||||
if isinstance(txt, bytes):
|
||||
txt = txt.decode()
|
||||
|
||||
if not isinstance(txt, str):
|
||||
raise TypeError('a string is required')
|
||||
|
||||
if not txt:
|
||||
raise Base58Error('string cannot be empty')
|
||||
|
||||
value = 0
|
||||
for c in txt:
|
||||
value = value * 58 + cls.char_value(c)
|
||||
|
||||
result = int_to_bytes(value)
|
||||
|
||||
# Prepend leading zero bytes if necessary
|
||||
count = 0
|
||||
for c in txt:
|
||||
if c != '1':
|
||||
break
|
||||
count += 1
|
||||
if count:
|
||||
result = bytes((0,)) * count + result
|
||||
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def encode(cls, be_bytes):
|
||||
"""Converts a big-endian bytearray into a base58 string."""
|
||||
value = bytes_to_int(be_bytes)
|
||||
|
||||
txt = ''
|
||||
while value:
|
||||
value, mod = divmod(value, 58)
|
||||
txt += cls.chars[mod]
|
||||
|
||||
for byte in be_bytes:
|
||||
if byte != 0:
|
||||
break
|
||||
txt += '1'
|
||||
|
||||
return txt[::-1]
|
||||
|
||||
@classmethod
|
||||
def decode_check(cls, txt, hash_fn=double_sha256):
|
||||
""" Decodes a Base58Check-encoded string to a payload. The version prefixes it. """
|
||||
be_bytes = cls.decode(txt)
|
||||
result, check = be_bytes[:-4], be_bytes[-4:]
|
||||
if check != hash_fn(result)[:4]:
|
||||
raise Base58Error(f'invalid base 58 checksum for {txt}')
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def encode_check(cls, payload, hash_fn=double_sha256):
|
||||
""" Encodes a payload bytearray (which includes the version byte(s))
|
||||
into a Base58Check string."""
|
||||
be_bytes = payload + hash_fn(payload)[:4]
|
||||
return cls.encode(be_bytes)
|
||||
@@ -1,253 +0,0 @@
|
||||
from coincurve import PublicKey, PrivateKey as _PrivateKey
|
||||
|
||||
from lbry.crypto.hash import hmac_sha512, hash160, double_sha256
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.utils import cachedproperty
|
||||
|
||||
|
||||
class DerivationError(Exception):
|
||||
""" Raised when an invalid derivation occurs. """
|
||||
|
||||
|
||||
class _KeyBase:
|
||||
""" A BIP32 Key, public or private. """
|
||||
|
||||
def __init__(self, ledger, chain_code, n, depth, parent):
|
||||
if not isinstance(chain_code, (bytes, bytearray)):
|
||||
raise TypeError('chain code must be raw bytes')
|
||||
if len(chain_code) != 32:
|
||||
raise ValueError('invalid chain code')
|
||||
if not 0 <= n < 1 << 32:
|
||||
raise ValueError('invalid child number')
|
||||
if not 0 <= depth < 256:
|
||||
raise ValueError('invalid depth')
|
||||
if parent is not None:
|
||||
if not isinstance(parent, type(self)):
|
||||
raise TypeError('parent key has bad type')
|
||||
self.ledger = ledger
|
||||
self.chain_code = chain_code
|
||||
self.n = n
|
||||
self.depth = depth
|
||||
self.parent = parent
|
||||
|
||||
def _hmac_sha512(self, msg):
|
||||
""" Use SHA-512 to provide an HMAC, returned as a pair of 32-byte objects. """
|
||||
hmac = hmac_sha512(self.chain_code, msg)
|
||||
return hmac[:32], hmac[32:]
|
||||
|
||||
def _extended_key(self, ver_bytes, raw_serkey):
|
||||
""" Return the 78-byte extended key given prefix version bytes and serialized key bytes. """
|
||||
if not isinstance(ver_bytes, (bytes, bytearray)):
|
||||
raise TypeError('ver_bytes must be raw bytes')
|
||||
if len(ver_bytes) != 4:
|
||||
raise ValueError('ver_bytes must have length 4')
|
||||
if not isinstance(raw_serkey, (bytes, bytearray)):
|
||||
raise TypeError('raw_serkey must be raw bytes')
|
||||
if len(raw_serkey) != 33:
|
||||
raise ValueError('raw_serkey must have length 33')
|
||||
|
||||
return (ver_bytes + bytes((self.depth,))
|
||||
+ self.parent_fingerprint() + self.n.to_bytes(4, 'big')
|
||||
+ self.chain_code + raw_serkey)
|
||||
|
||||
def identifier(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def extended_key(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def fingerprint(self):
|
||||
""" Return the key's fingerprint as 4 bytes. """
|
||||
return self.identifier()[:4]
|
||||
|
||||
def parent_fingerprint(self):
|
||||
""" Return the parent key's fingerprint as 4 bytes. """
|
||||
return self.parent.fingerprint() if self.parent else bytes((0,)*4)
|
||||
|
||||
def extended_key_string(self):
|
||||
""" Return an extended key as a base58 string. """
|
||||
return Base58.encode_check(self.extended_key())
|
||||
|
||||
|
||||
class PubKey(_KeyBase):
|
||||
""" A BIP32 public key. """
|
||||
|
||||
def __init__(self, ledger, pubkey, chain_code, n, depth, parent=None):
|
||||
super().__init__(ledger, chain_code, n, depth, parent)
|
||||
if isinstance(pubkey, PublicKey):
|
||||
self.verifying_key = pubkey
|
||||
else:
|
||||
self.verifying_key = self._verifying_key_from_pubkey(pubkey)
|
||||
|
||||
@classmethod
|
||||
def _verifying_key_from_pubkey(cls, pubkey):
|
||||
""" Converts a 33-byte compressed pubkey into an PublicKey object. """
|
||||
if not isinstance(pubkey, (bytes, bytearray)):
|
||||
raise TypeError('pubkey must be raw bytes')
|
||||
if len(pubkey) != 33:
|
||||
raise ValueError('pubkey must be 33 bytes')
|
||||
if pubkey[0] not in (2, 3):
|
||||
raise ValueError('invalid pubkey prefix byte')
|
||||
return PublicKey(pubkey)
|
||||
|
||||
@cachedproperty
|
||||
def pubkey_bytes(self):
|
||||
""" Return the compressed public key as 33 bytes. """
|
||||
return self.verifying_key.format(True)
|
||||
|
||||
@cachedproperty
|
||||
def address(self):
|
||||
""" The public key as a P2PKH address. """
|
||||
return self.ledger.public_key_to_address(self.pubkey_bytes)
|
||||
|
||||
def ec_point(self):
|
||||
return self.verifying_key.point()
|
||||
|
||||
def child(self, n: int):
|
||||
""" Return the derived child extended pubkey at index N. """
|
||||
if not 0 <= n < (1 << 31):
|
||||
raise ValueError('invalid BIP32 public key child number')
|
||||
|
||||
msg = self.pubkey_bytes + n.to_bytes(4, 'big')
|
||||
L_b, R_b = self._hmac_sha512(msg) # pylint: disable=invalid-name
|
||||
derived_key = self.verifying_key.add(L_b)
|
||||
return PubKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
|
||||
|
||||
def identifier(self):
|
||||
""" Return the key's identifier as 20 bytes. """
|
||||
return hash160(self.pubkey_bytes)
|
||||
|
||||
def extended_key(self):
|
||||
""" Return a raw extended public key. """
|
||||
return self._extended_key(
|
||||
self.ledger.extended_public_key_prefix,
|
||||
self.pubkey_bytes
|
||||
)
|
||||
|
||||
|
||||
class PrivateKey(_KeyBase):
|
||||
"""A BIP32 private key."""
|
||||
|
||||
HARDENED = 1 << 31
|
||||
|
||||
def __init__(self, ledger, privkey, chain_code, n, depth, parent=None):
|
||||
super().__init__(ledger, chain_code, n, depth, parent)
|
||||
if isinstance(privkey, _PrivateKey):
|
||||
self.signing_key = privkey
|
||||
else:
|
||||
self.signing_key = self._signing_key_from_privkey(privkey)
|
||||
|
||||
@classmethod
|
||||
def _signing_key_from_privkey(cls, private_key):
|
||||
""" Converts a 32-byte private key into an coincurve.PrivateKey object. """
|
||||
return _PrivateKey.from_int(PrivateKey._private_key_secret_exponent(private_key))
|
||||
|
||||
@classmethod
|
||||
def _private_key_secret_exponent(cls, private_key):
|
||||
""" Return the private key as a secret exponent if it is a valid private key. """
|
||||
if not isinstance(private_key, (bytes, bytearray)):
|
||||
raise TypeError('private key must be raw bytes')
|
||||
if len(private_key) != 32:
|
||||
raise ValueError('private key must be 32 bytes')
|
||||
return int.from_bytes(private_key, 'big')
|
||||
|
||||
@classmethod
|
||||
def from_seed(cls, ledger, seed):
|
||||
# This hard-coded message string seems to be coin-independent...
|
||||
hmac = hmac_sha512(b'Bitcoin seed', seed)
|
||||
privkey, chain_code = hmac[:32], hmac[32:]
|
||||
return cls(ledger, privkey, chain_code, 0, 0)
|
||||
|
||||
@cachedproperty
|
||||
def private_key_bytes(self):
|
||||
""" Return the serialized private key (no leading zero byte). """
|
||||
return self.signing_key.secret
|
||||
|
||||
@cachedproperty
|
||||
def public_key(self):
|
||||
""" Return the corresponding extended public key. """
|
||||
verifying_key = self.signing_key.public_key
|
||||
parent_pubkey = self.parent.public_key if self.parent else None
|
||||
return PubKey(self.ledger, verifying_key, self.chain_code, self.n, self.depth,
|
||||
parent_pubkey)
|
||||
|
||||
def ec_point(self):
|
||||
return self.public_key.ec_point()
|
||||
|
||||
def secret_exponent(self):
|
||||
""" Return the private key as a secret exponent. """
|
||||
return self.signing_key.to_int()
|
||||
|
||||
def wif(self):
|
||||
""" Return the private key encoded in Wallet Import Format. """
|
||||
return self.ledger.private_key_to_wif(self.private_key_bytes)
|
||||
|
||||
def address(self):
|
||||
""" The public key as a P2PKH address. """
|
||||
return self.public_key.address
|
||||
|
||||
def child(self, n):
|
||||
""" Return the derived child extended private key at index N."""
|
||||
if not 0 <= n < (1 << 32):
|
||||
raise ValueError('invalid BIP32 private key child number')
|
||||
|
||||
if n >= self.HARDENED:
|
||||
serkey = b'\0' + self.private_key_bytes
|
||||
else:
|
||||
serkey = self.public_key.pubkey_bytes
|
||||
|
||||
msg = serkey + n.to_bytes(4, 'big')
|
||||
L_b, R_b = self._hmac_sha512(msg) # pylint: disable=invalid-name
|
||||
derived_key = self.signing_key.add(L_b)
|
||||
return PrivateKey(self.ledger, derived_key, R_b, n, self.depth + 1, self)
|
||||
|
||||
def sign(self, data):
|
||||
""" Produce a signature for piece of data by double hashing it and signing the hash. """
|
||||
return self.signing_key.sign(data, hasher=double_sha256)
|
||||
|
||||
def identifier(self):
|
||||
"""Return the key's identifier as 20 bytes."""
|
||||
return self.public_key.identifier()
|
||||
|
||||
def extended_key(self):
|
||||
"""Return a raw extended private key."""
|
||||
return self._extended_key(
|
||||
self.ledger.extended_private_key_prefix,
|
||||
b'\0' + self.private_key_bytes
|
||||
)
|
||||
|
||||
|
||||
def _from_extended_key(ledger, ekey):
|
||||
"""Return a PubKey or PrivateKey from an extended key raw bytes."""
|
||||
if not isinstance(ekey, (bytes, bytearray)):
|
||||
raise TypeError('extended key must be raw bytes')
|
||||
if len(ekey) != 78:
|
||||
raise ValueError('extended key must have length 78')
|
||||
|
||||
depth = ekey[4]
|
||||
n = int.from_bytes(ekey[9:13], 'big')
|
||||
chain_code = ekey[13:45]
|
||||
|
||||
if ekey[:4] == ledger.extended_public_key_prefix:
|
||||
pubkey = ekey[45:]
|
||||
key = PubKey(ledger, pubkey, chain_code, n, depth)
|
||||
elif ekey[:4] == ledger.extended_private_key_prefix:
|
||||
if ekey[45] != 0:
|
||||
raise ValueError('invalid extended private key prefix byte')
|
||||
privkey = ekey[46:]
|
||||
key = PrivateKey(ledger, privkey, chain_code, n, depth)
|
||||
else:
|
||||
raise ValueError('version bytes unrecognised')
|
||||
|
||||
return key
|
||||
|
||||
|
||||
def from_extended_key_string(ledger, ekey_str):
|
||||
"""Given an extended key string, such as
|
||||
|
||||
xpub6BsnM1W2Y7qLMiuhi7f7dbAwQZ5Cz5gYJCRzTNainXzQXYjFwtuQXHd
|
||||
3qfi3t3KJtHxshXezfjft93w4UE7BGMtKwhqEHae3ZA7d823DVrL
|
||||
|
||||
return a PubKey or PrivateKey.
|
||||
"""
|
||||
return _from_extended_key(ledger, Base58.decode_check(ekey_str))
|
||||
@@ -1,71 +0,0 @@
|
||||
import os
|
||||
import base64
|
||||
import typing
|
||||
from cryptography.hazmat.primitives.kdf.scrypt import Scrypt
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, modes
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from cryptography.hazmat.primitives.padding import PKCS7
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
|
||||
from lbry.error import InvalidPasswordError
|
||||
from lbry.crypto.hash import double_sha256
|
||||
|
||||
|
||||
def aes_encrypt(secret: str, value: str, init_vector: bytes = None) -> str:
|
||||
if init_vector is not None:
|
||||
assert len(init_vector) == 16
|
||||
else:
|
||||
init_vector = os.urandom(16)
|
||||
key = double_sha256(secret.encode())
|
||||
encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor()
|
||||
padder = PKCS7(AES.block_size).padder()
|
||||
padded_data = padder.update(value.encode()) + padder.finalize()
|
||||
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
|
||||
return base64.b64encode(init_vector + encrypted_data).decode()
|
||||
|
||||
|
||||
def aes_decrypt(secret: str, value: str) -> typing.Tuple[str, bytes]:
|
||||
try:
|
||||
data = base64.b64decode(value.encode())
|
||||
key = double_sha256(secret.encode())
|
||||
init_vector, data = data[:16], data[16:]
|
||||
decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor()
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
result = unpadder.update(decryptor.update(data)) + unpadder.finalize()
|
||||
return result.decode(), init_vector
|
||||
except UnicodeDecodeError:
|
||||
raise InvalidPasswordError()
|
||||
except ValueError as e:
|
||||
if e.args[0] == 'Invalid padding bytes.':
|
||||
raise InvalidPasswordError()
|
||||
raise
|
||||
|
||||
|
||||
def better_aes_encrypt(secret: str, value: bytes) -> bytes:
|
||||
init_vector = os.urandom(16)
|
||||
key = scrypt(secret.encode(), salt=init_vector)
|
||||
encryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).encryptor()
|
||||
padder = PKCS7(AES.block_size).padder()
|
||||
padded_data = padder.update(value) + padder.finalize()
|
||||
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
|
||||
return base64.b64encode(b's:8192:16:1:' + init_vector + encrypted_data)
|
||||
|
||||
|
||||
def better_aes_decrypt(secret: str, value: bytes) -> bytes:
|
||||
try:
|
||||
data = base64.b64decode(value)
|
||||
_, scryp_n, scrypt_r, scrypt_p, data = data.split(b':', maxsplit=4)
|
||||
init_vector, data = data[:16], data[16:]
|
||||
key = scrypt(secret.encode(), init_vector, int(scryp_n), int(scrypt_r), int(scrypt_p))
|
||||
decryptor = Cipher(AES(key), modes.CBC(init_vector), default_backend()).decryptor()
|
||||
unpadder = PKCS7(AES.block_size).unpadder()
|
||||
return unpadder.update(decryptor.update(data)) + unpadder.finalize()
|
||||
except ValueError as e:
|
||||
if e.args[0] == 'Invalid padding bytes.':
|
||||
raise InvalidPasswordError()
|
||||
raise
|
||||
|
||||
|
||||
def scrypt(passphrase, salt, scrypt_n=1<<13, scrypt_r=16, scrypt_p=1):
|
||||
kdf = Scrypt(salt, length=32, n=scrypt_n, r=scrypt_r, p=scrypt_p, backend=default_backend())
|
||||
return kdf.derive(passphrase)
|
||||
@@ -1,47 +0,0 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
|
||||
def sha256(x):
|
||||
""" Simple wrapper of hashlib sha256. """
|
||||
return hashlib.sha256(x).digest()
|
||||
|
||||
|
||||
def sha512(x):
|
||||
""" Simple wrapper of hashlib sha512. """
|
||||
return hashlib.sha512(x).digest()
|
||||
|
||||
|
||||
def ripemd160(x):
|
||||
""" Simple wrapper of hashlib ripemd160. """
|
||||
h = hashlib.new('ripemd160')
|
||||
h.update(x)
|
||||
return h.digest()
|
||||
|
||||
|
||||
def double_sha256(x):
|
||||
""" SHA-256 of SHA-256, as used extensively in bitcoin. """
|
||||
return sha256(sha256(x))
|
||||
|
||||
|
||||
def hmac_sha512(key, msg):
|
||||
""" Use SHA-512 to provide an HMAC. """
|
||||
return hmac.new(key, msg, hashlib.sha512).digest()
|
||||
|
||||
|
||||
def hash160(x):
|
||||
""" RIPEMD-160 of SHA-256.
|
||||
Used to make bitcoin addresses from pubkeys. """
|
||||
return ripemd160(sha256(x))
|
||||
|
||||
|
||||
def hash_to_hex_str(x: bytes) -> str:
|
||||
""" Convert a big-endian binary hash to displayed hex string.
|
||||
Display form of a binary hash is reversed and converted to hex. """
|
||||
return hexlify(x[::-1])
|
||||
|
||||
|
||||
def hex_str_to_hash(x: str) -> bytes:
|
||||
""" Convert a displayed hex string to a binary hash. """
|
||||
return unhexlify(x)[::-1]
|
||||
@@ -1,13 +0,0 @@
|
||||
from binascii import unhexlify, hexlify
|
||||
|
||||
|
||||
def bytes_to_int(be_bytes):
|
||||
""" Interprets a big-endian sequence of bytes as an integer. """
|
||||
return int(hexlify(be_bytes), 16)
|
||||
|
||||
|
||||
def int_to_bytes(value):
|
||||
""" Converts an integer to a big-endian sequence of bytes. """
|
||||
length = (value.bit_length() + 7) // 8
|
||||
s = '%x' % value
|
||||
return unhexlify(('0' * (len(s) % 2) + s).zfill(length * 2))
|
||||
@@ -1,5 +0,0 @@
|
||||
from .database import Database, Result
|
||||
from .constants import (
|
||||
TXO_TYPES, SPENDABLE_TYPE_CODES,
|
||||
CLAIM_TYPE_CODES, CLAIM_TYPE_NAMES
|
||||
)
|
||||
@@ -1,73 +0,0 @@
|
||||
MAX_QUERY_VARIABLES = 900
|
||||
|
||||
TXO_TYPES = {
|
||||
"other": 0,
|
||||
"stream": 1,
|
||||
"channel": 2,
|
||||
"support": 3,
|
||||
"purchase": 4,
|
||||
"collection": 5,
|
||||
"repost": 6,
|
||||
}
|
||||
|
||||
CLAIM_TYPE_NAMES = [
|
||||
'stream',
|
||||
'channel',
|
||||
'collection',
|
||||
'repost',
|
||||
]
|
||||
|
||||
CONTENT_TYPE_NAMES = [
|
||||
name for name in CLAIM_TYPE_NAMES if name != "channel"
|
||||
]
|
||||
|
||||
CLAIM_TYPE_CODES = [
|
||||
TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
|
||||
]
|
||||
|
||||
CONTENT_TYPE_CODES = [
|
||||
TXO_TYPES[name] for name in CONTENT_TYPE_NAMES
|
||||
]
|
||||
|
||||
SPENDABLE_TYPE_CODES = [
|
||||
TXO_TYPES['other'],
|
||||
TXO_TYPES['purchase']
|
||||
]
|
||||
|
||||
STREAM_TYPES = {
|
||||
'video': 1,
|
||||
'audio': 2,
|
||||
'image': 3,
|
||||
'document': 4,
|
||||
'binary': 5,
|
||||
'model': 6
|
||||
}
|
||||
|
||||
MATURE_TAGS = (
|
||||
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
|
||||
)
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
|
||||
SEARCH_INTEGER_PARAMS = {
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
|
||||
'tx_position', 'channel_join', 'reposted',
|
||||
'amount', 'staked_amount', 'support_amount',
|
||||
'trend_group', 'trend_mixed', 'trend_local', 'trend_global',
|
||||
}
|
||||
|
||||
SEARCH_PARAMS = {
|
||||
'name', 'text', 'claim_id', 'claim_ids', 'txid', 'nout', 'channel', 'channel_ids', 'not_channel_ids',
|
||||
'public_key_id', 'claim_type', 'stream_types', 'media_types', 'fee_currency',
|
||||
'has_channel_signature', 'signature_valid',
|
||||
'any_tags', 'all_tags', 'not_tags', 'reposted_claim_id',
|
||||
'any_locations', 'all_locations', 'not_locations',
|
||||
'any_languages', 'all_languages', 'not_languages',
|
||||
'is_controlling', 'limit', 'offset', 'order_by',
|
||||
'no_totals',
|
||||
} | SEARCH_INTEGER_PARAMS
|
||||
|
||||
SEARCH_ORDER_FIELDS = {
|
||||
'name', 'claim_hash', 'claim_id'
|
||||
} | SEARCH_INTEGER_PARAMS
|
||||
@@ -1,402 +0,0 @@
|
||||
import os
|
||||
import asyncio
|
||||
import tempfile
|
||||
import multiprocessing as mp
|
||||
from binascii import hexlify
|
||||
from typing import List, Optional, Iterable, Iterator, TypeVar, Generic, TYPE_CHECKING, Dict, Tuple
|
||||
from concurrent.futures import Executor, ThreadPoolExecutor, ProcessPoolExecutor
|
||||
from functools import partial
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from lbry.event import EventController
|
||||
from lbry.crypto.bip32 import PubKey
|
||||
from lbry.blockchain.transaction import Transaction, Output
|
||||
from .constants import TXO_TYPES, CLAIM_TYPE_CODES
|
||||
from .query_context import initialize, uninitialize, ProgressPublisher
|
||||
from . import queries as q
|
||||
from . import sync
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lbry.blockchain.ledger import Ledger
|
||||
|
||||
|
||||
def clean_wallet_account_ids(constraints):
|
||||
wallet = constraints.pop('wallet', None)
|
||||
account = constraints.pop('account', None)
|
||||
accounts = constraints.pop('accounts', [])
|
||||
if account and not accounts:
|
||||
accounts = [account]
|
||||
if wallet:
|
||||
constraints['wallet_account_ids'] = [account.id for account in wallet.accounts]
|
||||
if not accounts:
|
||||
accounts = wallet.accounts
|
||||
if accounts:
|
||||
constraints['account_ids'] = [account.id for account in accounts]
|
||||
|
||||
|
||||
async def add_channel_keys_to_txo_results(accounts: List, txos: Iterable[Output]):
|
||||
sub_channels = set()
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.claim.is_channel:
|
||||
for account in accounts:
|
||||
private_key = await account.get_channel_private_key(
|
||||
txo.claim.channel.public_key_bytes
|
||||
)
|
||||
if private_key:
|
||||
txo.private_key = private_key
|
||||
break
|
||||
if txo.channel is not None:
|
||||
sub_channels.add(txo.channel)
|
||||
if sub_channels:
|
||||
await add_channel_keys_to_txo_results(accounts, sub_channels)
|
||||
|
||||
ResultType = TypeVar('ResultType')
|
||||
|
||||
|
||||
class Result(Generic[ResultType]):
|
||||
|
||||
__slots__ = 'rows', 'total', 'censor'
|
||||
|
||||
def __init__(self, rows: List[ResultType], total, censor=None):
|
||||
self.rows = rows
|
||||
self.total = total
|
||||
self.censor = censor
|
||||
|
||||
def __getitem__(self, item: int) -> ResultType:
|
||||
return self.rows[item]
|
||||
|
||||
def __iter__(self) -> Iterator[ResultType]:
|
||||
return iter(self.rows)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.rows)
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.rows)
|
||||
|
||||
|
||||
class Database:
|
||||
|
||||
def __init__(self, ledger: 'Ledger'):
|
||||
self.url = ledger.conf.db_url_or_default
|
||||
self.ledger = ledger
|
||||
self.workers = self._normalize_worker_processes(ledger.conf.workers)
|
||||
self.executor: Optional[Executor] = None
|
||||
self.message_queue = mp.Queue()
|
||||
self.stop_event = mp.Event()
|
||||
self._on_progress_controller = EventController()
|
||||
self.on_progress = self._on_progress_controller.stream
|
||||
self.progress_publisher = ProgressPublisher(
|
||||
self.message_queue, self._on_progress_controller
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_worker_processes(workers):
|
||||
if workers == 0:
|
||||
return os.cpu_count()
|
||||
elif workers > 0:
|
||||
return workers
|
||||
return 1
|
||||
|
||||
@classmethod
|
||||
def temp_from_url_regtest(cls, db_url, lbrycrd_config=None):
|
||||
from lbry import Config, RegTestLedger # pylint: disable=import-outside-toplevel
|
||||
directory = tempfile.mkdtemp()
|
||||
if lbrycrd_config:
|
||||
conf = lbrycrd_config
|
||||
conf.data_dir = directory
|
||||
conf.download_dir = directory
|
||||
conf.wallet_dir = directory
|
||||
else:
|
||||
conf = Config.with_same_dir(directory)
|
||||
conf.set(blockchain="regtest", db_url=db_url)
|
||||
ledger = RegTestLedger(conf)
|
||||
return cls(ledger)
|
||||
|
||||
@classmethod
|
||||
def temp_sqlite_regtest(cls, lbrycrd_config=None):
|
||||
from lbry import Config, RegTestLedger # pylint: disable=import-outside-toplevel
|
||||
directory = tempfile.mkdtemp()
|
||||
if lbrycrd_config:
|
||||
conf = lbrycrd_config
|
||||
conf.data_dir = directory
|
||||
conf.download_dir = directory
|
||||
conf.wallet_dir = directory
|
||||
else:
|
||||
conf = Config.with_same_dir(directory).set(blockchain="regtest")
|
||||
ledger = RegTestLedger(conf)
|
||||
return cls(ledger)
|
||||
|
||||
@classmethod
|
||||
def temp_sqlite(cls):
|
||||
from lbry import Config, Ledger # pylint: disable=import-outside-toplevel
|
||||
conf = Config.with_same_dir(tempfile.mkdtemp())
|
||||
return cls(Ledger(conf))
|
||||
|
||||
@classmethod
|
||||
def from_url(cls, db_url):
|
||||
from lbry import Config, Ledger # pylint: disable=import-outside-toplevel
|
||||
return cls(Ledger(Config.with_null_dir().set(db_url=db_url)))
|
||||
|
||||
@classmethod
|
||||
def in_memory(cls):
|
||||
return cls.from_url('sqlite:///:memory:')
|
||||
|
||||
def sync_create(self, name):
|
||||
engine = create_engine(self.url)
|
||||
db = engine.connect()
|
||||
db.execute(text("COMMIT"))
|
||||
db.execute(text(f"CREATE DATABASE {name}"))
|
||||
|
||||
async def create(self, name):
|
||||
return await asyncio.get_running_loop().run_in_executor(None, self.sync_create, name)
|
||||
|
||||
def sync_drop(self, name):
|
||||
engine = create_engine(self.url)
|
||||
db = engine.connect()
|
||||
db.execute(text("COMMIT"))
|
||||
db.execute(text(f"DROP DATABASE IF EXISTS {name}"))
|
||||
|
||||
async def drop(self, name):
|
||||
return await asyncio.get_running_loop().run_in_executor(None, self.sync_drop, name)
|
||||
|
||||
async def open(self):
|
||||
assert self.executor is None, "Database already open."
|
||||
self.progress_publisher.start()
|
||||
kwargs = {
|
||||
"initializer": initialize,
|
||||
"initargs": (
|
||||
self.ledger,
|
||||
self.message_queue, self.stop_event
|
||||
)
|
||||
}
|
||||
if self.workers > 1:
|
||||
self.executor = ProcessPoolExecutor(max_workers=self.workers, **kwargs)
|
||||
else:
|
||||
self.executor = ThreadPoolExecutor(max_workers=1, **kwargs)
|
||||
return await self.run(q.check_version_and_create_tables)
|
||||
|
||||
async def close(self):
|
||||
self.progress_publisher.stop()
|
||||
if self.executor is not None:
|
||||
if isinstance(self.executor, ThreadPoolExecutor):
|
||||
await self.run(uninitialize)
|
||||
self.executor.shutdown()
|
||||
self.executor = None
|
||||
# fixes "OSError: handle is closed"
|
||||
# seems to only happen when running in PyCharm
|
||||
# https://github.com/python/cpython/pull/6084#issuecomment-564585446
|
||||
# TODO: delete this in Python 3.8/3.9?
|
||||
from concurrent.futures.process import _threads_wakeups # pylint: disable=import-outside-toplevel
|
||||
_threads_wakeups.clear()
|
||||
|
||||
async def run(self, func, *args, **kwargs):
|
||||
if kwargs:
|
||||
clean_wallet_account_ids(kwargs)
|
||||
return await asyncio.get_running_loop().run_in_executor(
|
||||
self.executor, partial(func, *args, **kwargs)
|
||||
)
|
||||
|
||||
async def fetch_result(self, func, *args, **kwargs) -> Result:
|
||||
rows, total = await self.run(func, *args, **kwargs)
|
||||
return Result(rows, total)
|
||||
|
||||
async def execute(self, sql):
|
||||
return await self.run(q.execute, sql)
|
||||
|
||||
async def execute_sql_object(self, sql):
|
||||
return await self.run(q.execute_sql_object, sql)
|
||||
|
||||
async def execute_fetchall(self, sql):
|
||||
return await self.run(q.execute_fetchall, sql)
|
||||
|
||||
async def has_filters(self):
|
||||
return await self.run(q.has_filters)
|
||||
|
||||
async def has_claims(self):
|
||||
return await self.run(q.has_claims)
|
||||
|
||||
async def has_supports(self):
|
||||
return await self.run(q.has_supports)
|
||||
|
||||
async def has_wallet(self, wallet_id):
|
||||
return await self.run(q.has_wallet, wallet_id)
|
||||
|
||||
async def get_wallet(self, wallet_id: str):
|
||||
return await self.run(q.get_wallet, wallet_id)
|
||||
|
||||
async def add_wallet(self, wallet_id: str, data: str):
|
||||
return await self.run(q.add_wallet, wallet_id, data)
|
||||
|
||||
async def get_best_block_height(self) -> int:
|
||||
return await self.run(q.get_best_block_height)
|
||||
|
||||
async def process_all_things_after_sync(self):
|
||||
return await self.run(sync.process_all_things_after_sync)
|
||||
|
||||
async def get_block_headers(self, start_height: int, end_height: int = None):
|
||||
return await self.run(q.get_block_headers, start_height, end_height)
|
||||
|
||||
async def get_filters(self, start_height: int, end_height: int = None, granularity: int = 0):
|
||||
filters = []
|
||||
for row in await self.run(q.get_filters, start_height, end_height, granularity):
|
||||
record = {
|
||||
"height": row["height"],
|
||||
"filter": hexlify(row["address_filter"]).decode(),
|
||||
}
|
||||
if granularity == 0:
|
||||
record["txid"] = hexlify(row["tx_hash"][::-1]).decode()
|
||||
filters.append(record)
|
||||
return filters
|
||||
|
||||
async def get_missing_required_filters(self, height) -> Dict[int, Tuple[int, int]]:
|
||||
return await self.run(q.get_missing_required_filters, height)
|
||||
|
||||
async def get_missing_sub_filters_for_addresses(self, granularity, address_manager):
|
||||
return await self.run(q.get_missing_sub_filters_for_addresses, granularity, address_manager)
|
||||
|
||||
async def get_missing_tx_for_addresses(self, address_manager):
|
||||
return await self.run(q.get_missing_tx_for_addresses, address_manager)
|
||||
|
||||
async def insert_blocks(self, blocks):
|
||||
return await self.run(q.insert_blocks, blocks)
|
||||
|
||||
async def insert_block_filters(self, filters):
|
||||
return await self.run(q.insert_block_filters, filters)
|
||||
|
||||
async def insert_tx_filters(self, filters):
|
||||
return await self.run(q.insert_tx_filters, filters)
|
||||
|
||||
async def insert_transactions(self, txs):
|
||||
return await self.run(q.insert_transactions, txs)
|
||||
|
||||
async def update_address_used_times(self, addresses):
|
||||
return await self.run(q.update_address_used_times, addresses)
|
||||
|
||||
async def reserve_outputs(self, txos, is_reserved=True):
|
||||
txo_hashes = [txo.hash for txo in txos]
|
||||
if txo_hashes:
|
||||
return await self.run(
|
||||
q.reserve_outputs, txo_hashes, is_reserved
|
||||
)
|
||||
|
||||
async def release_outputs(self, txos):
|
||||
return await self.reserve_outputs(txos, is_reserved=False)
|
||||
|
||||
async def release_tx(self, tx):
|
||||
return await self.release_outputs([txi.txo_ref.txo for txi in tx.inputs])
|
||||
|
||||
async def release_all_outputs(self, account):
|
||||
return await self.run(q.release_all_outputs, account.id)
|
||||
|
||||
async def get_balance(self, **constraints):
|
||||
return await self.run(q.get_balance, **constraints)
|
||||
|
||||
async def get_report(self, accounts):
|
||||
return await self.run(q.get_report, accounts=accounts)
|
||||
|
||||
async def get_addresses(self, **constraints) -> Result[dict]:
|
||||
addresses = await self.fetch_result(q.get_addresses, **constraints)
|
||||
if addresses and 'pubkey' in addresses[0]:
|
||||
for address in addresses:
|
||||
address['pubkey'] = PubKey(
|
||||
self.ledger, bytes(address.pop('pubkey')), bytes(address.pop('chain_code')),
|
||||
address.pop('n'), address.pop('depth')
|
||||
)
|
||||
return addresses
|
||||
|
||||
async def get_all_addresses(self):
|
||||
return await self.run(q.get_all_addresses)
|
||||
|
||||
async def get_address(self, **constraints):
|
||||
for address in await self.get_addresses(limit=1, **constraints):
|
||||
return address
|
||||
|
||||
async def add_keys(self, account, chain, pubkeys):
|
||||
return await self.run(q.add_keys, [{
|
||||
'account': account.id,
|
||||
'address': k.address,
|
||||
'chain': chain,
|
||||
'pubkey': k.pubkey_bytes,
|
||||
'chain_code': k.chain_code,
|
||||
'n': k.n,
|
||||
'depth': k.depth
|
||||
} for k in pubkeys])
|
||||
|
||||
async def generate_addresses_using_filters(self, best_height, allowed_gap, address_manager):
|
||||
return await self.run(
|
||||
q.generate_addresses_using_filters, best_height, allowed_gap, address_manager
|
||||
)
|
||||
|
||||
async def get_raw_transactions(self, **constraints):
|
||||
return await self.run(q.get_raw_transactions, **constraints)
|
||||
|
||||
async def get_transactions(self, **constraints) -> Result[Transaction]:
|
||||
return await self.fetch_result(q.get_transactions, **constraints)
|
||||
|
||||
async def get_transaction(self, **constraints) -> Optional[Transaction]:
|
||||
txs = await self.get_transactions(limit=1, **constraints)
|
||||
if txs:
|
||||
return txs[0]
|
||||
|
||||
async def get_purchases(self, **constraints) -> Result[Output]:
|
||||
return await self.fetch_result(q.get_purchases, **constraints)
|
||||
|
||||
async def search_claims(self, **constraints) -> Result[Output]:
|
||||
#assert set(constraints).issubset(SEARCH_PARAMS), \
|
||||
# f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
|
||||
claims, total, censor = await self.run(q.search_claims, **constraints)
|
||||
return Result(claims, total, censor)
|
||||
|
||||
async def protobuf_search_claims(self, **constraints) -> str:
|
||||
return await self.run(q.protobuf_search_claims, **constraints)
|
||||
|
||||
async def search_supports(self, **constraints) -> Result[Output]:
|
||||
return await self.fetch_result(q.search_supports, **constraints)
|
||||
|
||||
async def sum_supports(self, claim_hash, include_channel_content=False, exclude_own_supports=False) \
|
||||
-> Tuple[List[Dict], int]:
|
||||
return await self.run(q.sum_supports, claim_hash, include_channel_content, exclude_own_supports)
|
||||
|
||||
async def resolve(self, urls, **kwargs) -> Dict[str, Output]:
|
||||
return await self.run(q.resolve, urls, **kwargs)
|
||||
|
||||
async def protobuf_resolve(self, urls, **kwargs) -> str:
|
||||
return await self.run(q.protobuf_resolve, urls, **kwargs)
|
||||
|
||||
async def get_txo_sum(self, **constraints) -> int:
|
||||
return await self.run(q.get_txo_sum, **constraints)
|
||||
|
||||
async def get_txo_plot(self, **constraints) -> List[dict]:
|
||||
return await self.run(q.get_txo_plot, **constraints)
|
||||
|
||||
async def get_txos(self, **constraints) -> Result[Output]:
|
||||
txos = await self.fetch_result(q.get_txos, **constraints)
|
||||
if 'wallet' in constraints:
|
||||
await add_channel_keys_to_txo_results(constraints['wallet'].accounts, txos)
|
||||
return txos
|
||||
|
||||
async def get_utxos(self, **constraints) -> Result[Output]:
|
||||
return await self.get_txos(spent_height=0, **constraints)
|
||||
|
||||
async def get_supports(self, **constraints) -> Result[Output]:
|
||||
return await self.get_utxos(txo_type=TXO_TYPES['support'], **constraints)
|
||||
|
||||
async def get_claims(self, **constraints) -> Result[Output]:
|
||||
if 'txo_type' not in constraints:
|
||||
constraints['txo_type__in'] = CLAIM_TYPE_CODES
|
||||
txos = await self.fetch_result(q.get_txos, **constraints)
|
||||
if 'wallet' in constraints:
|
||||
await add_channel_keys_to_txo_results(constraints['wallet'].accounts, txos)
|
||||
return txos
|
||||
|
||||
async def get_streams(self, **constraints) -> Result[Output]:
|
||||
return await self.get_claims(txo_type=TXO_TYPES['stream'], **constraints)
|
||||
|
||||
async def get_channels(self, **constraints) -> Result[Output]:
|
||||
return await self.get_claims(txo_type=TXO_TYPES['channel'], **constraints)
|
||||
|
||||
async def get_collections(self, **constraints) -> Result[Output]:
|
||||
return await self.get_claims(txo_type=TXO_TYPES['collection'], **constraints)
|
||||
@@ -1,7 +0,0 @@
|
||||
from .base import *
|
||||
from .txio import *
|
||||
from .search import *
|
||||
from .resolve import *
|
||||
from .address import *
|
||||
from .wallet import *
|
||||
from .filters import *
|
||||
@@ -1,205 +0,0 @@
|
||||
import logging
|
||||
from typing import Tuple, List, Set, Iterator, Optional
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from lbry.crypto.hash import hash160
|
||||
from lbry.crypto.bip32 import PubKey
|
||||
|
||||
from ..utils import query
|
||||
from ..query_context import context
|
||||
from ..tables import TXO, PubkeyAddress, AccountAddress
|
||||
from .filters import (
|
||||
get_filter_matchers, get_filter_matchers_at_granularity, has_filter_range,
|
||||
get_tx_matchers_for_missing_txs,
|
||||
)
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DatabaseAddressIterator:
|
||||
|
||||
def __init__(self, account_id, chain):
|
||||
self.account_id = account_id
|
||||
self.chain = chain
|
||||
self.n = -1
|
||||
|
||||
@staticmethod
|
||||
def get_sql(account_id, chain):
|
||||
return (
|
||||
select(
|
||||
AccountAddress.c.pubkey,
|
||||
AccountAddress.c.n
|
||||
).where(
|
||||
(AccountAddress.c.account == account_id) &
|
||||
(AccountAddress.c.chain == chain)
|
||||
).order_by(AccountAddress.c.n)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_address_hash_bytes(account_id, chain):
|
||||
return [
|
||||
bytearray(hash160(row['pubkey'])) for row in context().fetchall(
|
||||
DatabaseAddressIterator.get_sql(account_id, chain)
|
||||
)
|
||||
]
|
||||
|
||||
def __iter__(self) -> Iterator[Tuple[bytes, int, bool]]:
|
||||
with context().connect_streaming() as c:
|
||||
sql = self.get_sql(self.account_id, self.chain)
|
||||
for row in c.execute(sql):
|
||||
self.n = row['n']
|
||||
yield hash160(row['pubkey']), self.n, False
|
||||
|
||||
|
||||
class PersistingAddressIterator(DatabaseAddressIterator):
|
||||
|
||||
def __init__(self, account_id, chain, pubkey_bytes, chain_code, depth):
|
||||
super().__init__(account_id, chain)
|
||||
self.pubkey_bytes = pubkey_bytes
|
||||
self.chain_code = chain_code
|
||||
self.depth = depth
|
||||
self.pubkey_buffer = []
|
||||
|
||||
def flush(self):
|
||||
if self.pubkey_buffer:
|
||||
add_keys([{
|
||||
'account': self.account_id,
|
||||
'address': k.address,
|
||||
'chain': self.chain,
|
||||
'pubkey': k.pubkey_bytes,
|
||||
'chain_code': k.chain_code,
|
||||
'n': k.n,
|
||||
'depth': k.depth
|
||||
} for k in self.pubkey_buffer])
|
||||
self.pubkey_buffer.clear()
|
||||
|
||||
def __enter__(self) -> 'PersistingAddressIterator':
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.flush()
|
||||
|
||||
def __iter__(self) -> Iterator[Tuple[bytes, int, bool]]:
|
||||
yield from super().__iter__()
|
||||
pubkey = PubKey(context().ledger, self.pubkey_bytes, self.chain_code, 0, self.depth)
|
||||
while True:
|
||||
self.n += 1
|
||||
pubkey_child = pubkey.child(self.n)
|
||||
self.pubkey_buffer.append(pubkey_child)
|
||||
if len(self.pubkey_buffer) >= 900:
|
||||
self.flush()
|
||||
yield hash160(pubkey_child.pubkey_bytes), self.n, True
|
||||
|
||||
|
||||
class BatchAddressIterator:
|
||||
|
||||
def __init__(self, iterator: PersistingAddressIterator, size):
|
||||
self.iterator = iterator
|
||||
self.size = size
|
||||
|
||||
def __iter__(self) -> Iterator[bytearray]:
|
||||
i = iter(self.iterator)
|
||||
while True:
|
||||
yield [bytearray(next(i)[0]) for _ in range(self.size)]
|
||||
|
||||
|
||||
def generate_addresses_using_filters(best_height, allowed_gap, address_manager) -> Set:
|
||||
need, have = set(), set()
|
||||
matchers = get_filter_matchers(best_height)
|
||||
with PersistingAddressIterator(*address_manager) as address_iterator:
|
||||
for addresses in BatchAddressIterator(address_iterator, allowed_gap):
|
||||
has_match = False
|
||||
for matcher, filter_range in matchers:
|
||||
if matcher.MatchAny(addresses):
|
||||
has_match = True
|
||||
if filter_range not in need and filter_range not in have:
|
||||
if has_filter_range(*filter_range):
|
||||
have.add(filter_range)
|
||||
else:
|
||||
need.add(filter_range)
|
||||
if not has_match:
|
||||
break
|
||||
return need
|
||||
|
||||
|
||||
def get_missing_sub_filters_for_addresses(granularity, address_manager):
|
||||
need = set()
|
||||
filters = get_filter_matchers_at_granularity(granularity)
|
||||
addresses = DatabaseAddressIterator.get_address_hash_bytes(*address_manager)
|
||||
for matcher, filter_range in filters:
|
||||
if matcher.MatchAny(addresses) and not has_filter_range(*filter_range):
|
||||
need.add(filter_range)
|
||||
return need
|
||||
|
||||
|
||||
def get_missing_tx_for_addresses(address_manager):
|
||||
need = set()
|
||||
filters = get_tx_matchers_for_missing_txs()
|
||||
print(f' loaded tx filters ({len(filters)})')
|
||||
addresses = DatabaseAddressIterator.get_address_hash_bytes(*address_manager)
|
||||
print(f' loaded addresses ({len(addresses)})')
|
||||
print(' matching...')
|
||||
for i, (tx_hash, matcher) in enumerate(filters):
|
||||
if i > 0 and i % 1000 == 0:
|
||||
print(f' {i} of {len(filters)} processed')
|
||||
if matcher.MatchAny(addresses):
|
||||
need.add(tx_hash)
|
||||
return need
|
||||
|
||||
|
||||
def update_address_used_times(addresses):
|
||||
context().execute(
|
||||
PubkeyAddress.update()
|
||||
.values(used_times=(
|
||||
select(func.count(TXO.c.address))
|
||||
.where((TXO.c.address == PubkeyAddress.c.address)),
|
||||
))
|
||||
.where(PubkeyAddress.c.address._in(addresses))
|
||||
)
|
||||
|
||||
|
||||
def select_addresses(cols, **constraints):
|
||||
return context().fetchall(query(
|
||||
[AccountAddress, PubkeyAddress],
|
||||
select(*cols).select_from(PubkeyAddress.join(AccountAddress)),
|
||||
**constraints
|
||||
))
|
||||
|
||||
|
||||
def get_addresses(cols=None, include_total=False, **constraints) -> Tuple[List[dict], Optional[int]]:
|
||||
if cols is None:
|
||||
cols = (
|
||||
PubkeyAddress.c.address,
|
||||
PubkeyAddress.c.used_times,
|
||||
AccountAddress.c.account,
|
||||
AccountAddress.c.chain,
|
||||
AccountAddress.c.pubkey,
|
||||
AccountAddress.c.chain_code,
|
||||
AccountAddress.c.n,
|
||||
AccountAddress.c.depth
|
||||
)
|
||||
return (
|
||||
select_addresses(cols, **constraints),
|
||||
get_address_count(**constraints) if include_total else None
|
||||
)
|
||||
|
||||
|
||||
def get_address_count(**constraints):
|
||||
count = select_addresses([func.count().label("total")], **constraints)
|
||||
return count[0]["total"] or 0
|
||||
|
||||
|
||||
def get_all_addresses():
|
||||
return [r["address"] for r in context().fetchall(select(PubkeyAddress.c.address))]
|
||||
|
||||
|
||||
def add_keys(pubkeys):
|
||||
c = context()
|
||||
current_limit = c.variable_limit // len(pubkeys[0]) # (overall limit) // (maximum on a query)
|
||||
for start in range(0, len(pubkeys), current_limit - 1):
|
||||
batch = pubkeys[start:(start + current_limit - 1)]
|
||||
c.execute(c.insert_or_ignore(PubkeyAddress).values([{'address': k['address']} for k in batch]))
|
||||
c.execute(c.insert_or_ignore(AccountAddress).values(batch))
|
||||
@@ -1,99 +0,0 @@
|
||||
from sqlalchemy import text, between
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from ..query_context import context
|
||||
from ..tables import (
|
||||
SCHEMA_VERSION, metadata, Version,
|
||||
Claim, Support, Block, TX,
|
||||
pg_add_account_address_constraints_and_indexes
|
||||
)
|
||||
|
||||
|
||||
def execute(sql):
|
||||
return context().execute(text(sql))
|
||||
|
||||
|
||||
def execute_sql_object(sql):
|
||||
return context().execute(sql)
|
||||
|
||||
|
||||
def execute_fetchall(sql):
|
||||
return context().fetchall(text(sql))
|
||||
|
||||
|
||||
def has_claims():
|
||||
return context().has_records(Claim)
|
||||
|
||||
|
||||
def has_supports():
|
||||
return context().has_records(Support)
|
||||
|
||||
|
||||
def get_best_block_height():
|
||||
return context().fetchmax(Block.c.height, -1)
|
||||
|
||||
|
||||
def insert_blocks(blocks):
|
||||
loader = context().get_bulk_loader()
|
||||
for block in blocks:
|
||||
loader.add_block(block)
|
||||
loader.flush(return_row_count_for_table=None)
|
||||
|
||||
|
||||
def get_block_headers(first, last=None):
|
||||
q = (
|
||||
select(
|
||||
Block.c.height,
|
||||
Block.c.block_hash,
|
||||
Block.c.previous_hash,
|
||||
Block.c.timestamp,
|
||||
)
|
||||
.select_from(Block)
|
||||
)
|
||||
if last is not None:
|
||||
query = (
|
||||
q.where(between(Block.c.height, first, last))
|
||||
.order_by(Block.c.height)
|
||||
)
|
||||
else:
|
||||
query = q.where(Block.c.height == first)
|
||||
rows = context().fetchall(query)
|
||||
for row in rows:
|
||||
row['block_hash'] = bytes(row['block_hash'])
|
||||
row['previous_hash'] = bytes(row['previous_hash'])
|
||||
return rows
|
||||
|
||||
|
||||
def insert_transactions(txs):
|
||||
loader = context().get_bulk_loader()
|
||||
for block_hash, tx in txs:
|
||||
loader.add_transaction(block_hash, tx)
|
||||
loader.flush(return_row_count_for_table=None)
|
||||
|
||||
|
||||
def check_version_and_create_tables():
|
||||
with context("db.connecting") as ctx:
|
||||
if ctx.has_table('version'):
|
||||
version = ctx.fetchone(select(Version.c.version).limit(1))
|
||||
if version and version['version'] == SCHEMA_VERSION:
|
||||
return
|
||||
metadata.drop_all(ctx.engine)
|
||||
metadata.create_all(ctx.engine)
|
||||
ctx.execute(Version.insert().values(version=SCHEMA_VERSION))
|
||||
for table in metadata.sorted_tables:
|
||||
disable_trigger_and_constraints(table.name)
|
||||
if ctx.is_postgres:
|
||||
for statement in pg_add_account_address_constraints_and_indexes:
|
||||
ctx.execute(text(statement))
|
||||
|
||||
|
||||
def disable_trigger_and_constraints(table_name):
|
||||
ctx = context()
|
||||
if ctx.is_postgres:
|
||||
ctx.execute(text(f"ALTER TABLE {table_name} DISABLE TRIGGER ALL;"))
|
||||
if table_name in ('tag', 'stake', 'block_filter', 'mempool_filter'):
|
||||
return
|
||||
if ctx.is_postgres:
|
||||
ctx.execute(text(
|
||||
f"ALTER TABLE {table_name} DROP CONSTRAINT {table_name}_pkey CASCADE;"
|
||||
))
|
||||
@@ -1,175 +0,0 @@
|
||||
from math import log10
|
||||
from typing import Dict, List, Tuple, Set, Optional
|
||||
|
||||
from sqlalchemy import between, func, or_
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from lbry.blockchain.block import PyBIP158, get_address_filter
|
||||
|
||||
from ..query_context import context
|
||||
from ..tables import BlockFilter, TXFilter, TX
|
||||
|
||||
|
||||
def has_filters():
|
||||
return context().has_records(BlockFilter)
|
||||
|
||||
|
||||
def get_sub_filter_range(granularity: int, height: int):
|
||||
end = height
|
||||
if granularity >= 3:
|
||||
end = height + 10**(granularity-1) * 9
|
||||
elif granularity == 2:
|
||||
end = height + 99
|
||||
return granularity - 1, height, end
|
||||
|
||||
|
||||
def has_filter_range(factor: int, start: int, end: int):
|
||||
if factor >= 1:
|
||||
filters = context().fetchtotal(
|
||||
(BlockFilter.c.factor == factor) &
|
||||
between(BlockFilter.c.height, start, end)
|
||||
)
|
||||
expected = 10 if factor >= 2 else 100
|
||||
return filters == expected
|
||||
elif factor == 0:
|
||||
tx_filters_count = context().fetchtotal(TXFilter.c.height == start)
|
||||
return tx_filters_count > 0
|
||||
|
||||
|
||||
def get_filters(start_height, end_height=None, granularity=0):
|
||||
assert granularity >= 0, "filter granularity must be 0 or positive number"
|
||||
if granularity == 0:
|
||||
query = (
|
||||
select(TXFilter.c.height, TXFilter.c.address_filter, TXFilter.c.tx_hash)
|
||||
.select_from(TXFilter)
|
||||
.where(between(TXFilter.c.height, start_height, end_height))
|
||||
.order_by(TXFilter.c.height)
|
||||
)
|
||||
else:
|
||||
factor = granularity if granularity < 100 else log10(granularity)
|
||||
if end_height is None:
|
||||
height_condition = (BlockFilter.c.height == start_height)
|
||||
elif end_height == -1:
|
||||
height_condition = (BlockFilter.c.height >= start_height)
|
||||
else:
|
||||
height_condition = between(BlockFilter.c.height, start_height, end_height)
|
||||
query = (
|
||||
select(BlockFilter.c.height, BlockFilter.c.address_filter)
|
||||
.select_from(BlockFilter)
|
||||
.where(height_condition & (BlockFilter.c.factor == factor))
|
||||
.order_by(BlockFilter.c.height)
|
||||
)
|
||||
return context().fetchall(query)
|
||||
|
||||
|
||||
def get_minimal_required_filter_ranges(height) -> Dict[int, Tuple[int, int]]:
|
||||
minimal = {}
|
||||
if height >= 100_000:
|
||||
minimal[5] = (0, ((height // 100_000)-1) * 100_000)
|
||||
if height >= 10_000:
|
||||
start = height - height % 100_000
|
||||
minimal[4] = (start, start + (((height - start) // 10_000) - 1) * 10_000)
|
||||
if height >= 1_000:
|
||||
start = height - height % 10_000
|
||||
minimal[3] = (start, start+(((height-start) // 1_000)-1) * 1_000)
|
||||
if height >= 100:
|
||||
start = height - height % 1_000
|
||||
minimal[2] = (start, start+(((height-start) // 100)-1) * 100)
|
||||
start = height - height % 100
|
||||
if start < height:
|
||||
minimal[1] = (start, height)
|
||||
return minimal
|
||||
|
||||
|
||||
def get_maximum_known_filters() -> Dict[str, Optional[int]]:
|
||||
query = select(
|
||||
select(func.max(BlockFilter.c.height))
|
||||
.where(BlockFilter.c.factor == 1)
|
||||
.scalar_subquery().label('1'),
|
||||
select(func.max(BlockFilter.c.height))
|
||||
.where(BlockFilter.c.factor == 2)
|
||||
.scalar_subquery().label('2'),
|
||||
select(func.max(BlockFilter.c.height))
|
||||
.where(BlockFilter.c.factor == 3)
|
||||
.scalar_subquery().label('3'),
|
||||
select(func.max(BlockFilter.c.height))
|
||||
.where(BlockFilter.c.factor == 4)
|
||||
.scalar_subquery().label('4'),
|
||||
select(func.max(BlockFilter.c.height))
|
||||
.where(BlockFilter.c.factor == 5)
|
||||
.scalar_subquery().label('5'),
|
||||
)
|
||||
return context().fetchone(query)
|
||||
|
||||
|
||||
def get_missing_required_filters(height) -> Set[Tuple[int, int, int]]:
|
||||
known_filters = get_maximum_known_filters()
|
||||
missing_filters = set()
|
||||
for granularity, (start, end) in get_minimal_required_filter_ranges(height).items():
|
||||
known_height = known_filters.get(str(granularity))
|
||||
if known_height is not None and known_height >= start:
|
||||
if granularity == 1:
|
||||
adjusted_height = known_height + 1
|
||||
else:
|
||||
adjusted_height = known_height + 10**granularity
|
||||
if adjusted_height <= end:
|
||||
missing_filters.add((granularity, adjusted_height, end))
|
||||
else:
|
||||
missing_filters.add((granularity, start, end))
|
||||
return missing_filters
|
||||
|
||||
|
||||
def get_filter_matchers(height) -> List[Tuple[PyBIP158, Tuple[int, int, int]]]:
|
||||
conditions = []
|
||||
for granularity, (start, end) in get_minimal_required_filter_ranges(height).items():
|
||||
conditions.append(
|
||||
(BlockFilter.c.factor == granularity) &
|
||||
between(BlockFilter.c.height, start, end)
|
||||
)
|
||||
query = (
|
||||
select(BlockFilter.c.factor, BlockFilter.c.height, BlockFilter.c.address_filter)
|
||||
.select_from(BlockFilter)
|
||||
.where(or_(*conditions))
|
||||
.order_by(BlockFilter.c.height.desc())
|
||||
)
|
||||
return [(
|
||||
get_address_filter(bf["address_filter"]),
|
||||
get_sub_filter_range(bf["factor"], bf["height"])
|
||||
) for bf in context().fetchall(query)]
|
||||
|
||||
|
||||
def get_filter_matchers_at_granularity(granularity) -> List[Tuple[PyBIP158, Tuple]]:
|
||||
query = (
|
||||
select(BlockFilter.c.height, BlockFilter.c.address_filter)
|
||||
.where(BlockFilter.c.factor == granularity)
|
||||
.order_by(BlockFilter.c.height.desc())
|
||||
)
|
||||
return [(
|
||||
get_address_filter(bf["address_filter"]),
|
||||
get_sub_filter_range(granularity, bf["height"])
|
||||
) for bf in context().fetchall(query)]
|
||||
|
||||
|
||||
def get_tx_matchers_for_missing_txs() -> List[Tuple[int, PyBIP158]]:
|
||||
query = (
|
||||
select(TXFilter.c.tx_hash, TXFilter.c.address_filter)
|
||||
.where(TXFilter.c.tx_hash.notin_(select(TX.c.tx_hash)))
|
||||
)
|
||||
return [
|
||||
(bf["tx_hash"], get_address_filter(bf["address_filter"]))
|
||||
for bf in context().fetchall(query)
|
||||
]
|
||||
|
||||
|
||||
def insert_block_filters(filters):
|
||||
loader = context().get_bulk_loader()
|
||||
for height, factor, address_filter in filters:
|
||||
loader.add_block_filter(height, factor, address_filter)
|
||||
loader.flush(return_row_count_for_table=None)
|
||||
|
||||
|
||||
def insert_tx_filters(filters):
|
||||
loader = context().get_bulk_loader()
|
||||
for tx_hash, height, address_filter in filters:
|
||||
loader.add_transaction_filter(tx_hash, height, address_filter)
|
||||
loader.flush(return_row_count_for_table=None)
|
||||
@@ -1,101 +0,0 @@
|
||||
import logging
|
||||
import itertools
|
||||
from typing import List, Dict
|
||||
|
||||
from lbry.schema.url import URL
|
||||
from lbry.schema.result import Outputs as ResultOutput
|
||||
from lbry.error import ResolveCensoredError
|
||||
from lbry.blockchain.transaction import Output
|
||||
from . import rows_to_txos
|
||||
|
||||
from ..query_context import context
|
||||
from .search import select_claims
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def resolve_claims(**constraints):
|
||||
censor = context().get_resolve_censor()
|
||||
rows = context().fetchall(select_claims(**constraints))
|
||||
rows = censor.apply(rows)
|
||||
return rows_to_txos(rows), censor
|
||||
|
||||
|
||||
def _get_referenced_rows(txo_rows: List[Output], censor_channels: List[bytes]):
|
||||
repost_hashes = set(txo.reposted_claim.claim_hash for txo in txo_rows if txo.reposted_claim)
|
||||
channel_hashes = set(itertools.chain(
|
||||
(txo.channel.claim_hash for txo in txo_rows if txo.channel),
|
||||
censor_channels
|
||||
))
|
||||
|
||||
reposted_txos = []
|
||||
if repost_hashes:
|
||||
reposted_txos = resolve_claims(**{'claim.claim_hash__in': repost_hashes})
|
||||
if reposted_txos:
|
||||
reposted_txos = reposted_txos[0]
|
||||
channel_hashes |= set(txo.channel.claim_hash for txo in reposted_txos if txo.channel)
|
||||
|
||||
channel_txos = []
|
||||
if channel_hashes:
|
||||
channel_txos = resolve_claims(**{'claim.claim_hash__in': channel_hashes})
|
||||
channel_txos = channel_txos[0] if channel_txos else []
|
||||
|
||||
# channels must come first for client side inflation to work properly
|
||||
return channel_txos + reposted_txos
|
||||
|
||||
|
||||
def protobuf_resolve(urls, **kwargs) -> str:
|
||||
txo_rows = [resolve_url(raw_url) for raw_url in urls]
|
||||
extra_txo_rows = _get_referenced_rows(
|
||||
[txo_row for txo_row in txo_rows if isinstance(txo_row, Output)],
|
||||
[txo.censor_hash for txo in txo_rows if isinstance(txo, ResolveCensoredError)]
|
||||
)
|
||||
return ResultOutput.to_base64(txo_rows, extra_txo_rows)
|
||||
|
||||
|
||||
def resolve(urls, **kwargs) -> Dict[str, Output]:
|
||||
return {url: resolve_url(url) for url in urls}
|
||||
|
||||
|
||||
def resolve_url(raw_url):
|
||||
try:
|
||||
url = URL.parse(raw_url)
|
||||
except ValueError as e:
|
||||
return e
|
||||
|
||||
channel = None
|
||||
|
||||
if url.has_channel:
|
||||
q = url.channel.to_dict()
|
||||
if set(q) == {'name'}:
|
||||
q['is_controlling'] = True
|
||||
else:
|
||||
q['order_by'] = ['^creation_height']
|
||||
matches, censor = resolve_claims(**q, limit=1)
|
||||
if matches:
|
||||
channel = matches[0]
|
||||
elif censor.censored:
|
||||
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
|
||||
elif not channel:
|
||||
return LookupError(f'Could not find channel in "{raw_url}".')
|
||||
|
||||
if url.has_stream:
|
||||
q = url.stream.to_dict()
|
||||
if channel is not None:
|
||||
q['order_by'] = ['^creation_height']
|
||||
q['channel_hash'] = channel.claim_hash
|
||||
q['is_signature_valid'] = True
|
||||
elif set(q) == {'name'}:
|
||||
q['is_controlling'] = True
|
||||
matches, censor = resolve_claims(**q, limit=1)
|
||||
if matches:
|
||||
stream = matches[0]
|
||||
stream.channel = channel
|
||||
return stream
|
||||
elif censor.censored:
|
||||
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
|
||||
else:
|
||||
return LookupError(f'Could not find claim at "{raw_url}".')
|
||||
|
||||
return channel
|
||||
@@ -1,472 +0,0 @@
|
||||
import struct
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
from binascii import unhexlify
|
||||
from typing import Tuple, List, Optional, Dict
|
||||
|
||||
from sqlalchemy import func, case, text
|
||||
from sqlalchemy.future import select, Select
|
||||
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.result import Censor, Outputs as ResultOutput
|
||||
from lbry.schema.url import normalize_name
|
||||
from lbry.blockchain.transaction import Output
|
||||
|
||||
from ..utils import query
|
||||
from ..query_context import context
|
||||
from ..tables import TX, TXO, Claim, Support, Trend, CensoredClaim
|
||||
from ..constants import (
|
||||
TXO_TYPES, STREAM_TYPES, ATTRIBUTE_ARRAY_MAX_LENGTH,
|
||||
SEARCH_INTEGER_PARAMS, SEARCH_ORDER_FIELDS
|
||||
)
|
||||
|
||||
from .txio import BASE_SELECT_TXO_COLUMNS, rows_to_txos
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
BASE_SELECT_SUPPORT_COLUMNS = BASE_SELECT_TXO_COLUMNS + [
|
||||
Support.c.channel_hash,
|
||||
Support.c.is_signature_valid,
|
||||
]
|
||||
|
||||
|
||||
def compat_layer(**constraints):
|
||||
# for old sdk, to be removed later
|
||||
replacements = {
|
||||
"effective_amount": "staked_amount",
|
||||
"trending_mixed": "trend_mixed",
|
||||
"trending_group": "trend_group",
|
||||
"trending_local": "trend_local"
|
||||
}
|
||||
for old_key, new_key in replacements.items():
|
||||
if old_key in constraints:
|
||||
constraints[new_key] = constraints.pop(old_key)
|
||||
order_by = constraints.get("order_by", [])
|
||||
if old_key in order_by:
|
||||
constraints["order_by"] = [order_key if order_key != old_key else new_key for order_key in order_by]
|
||||
return constraints
|
||||
|
||||
|
||||
def select_supports(cols: List = None, **constraints) -> Select:
|
||||
if cols is None:
|
||||
cols = BASE_SELECT_SUPPORT_COLUMNS
|
||||
joins = Support.join(TXO, ).join(TX)
|
||||
return query([Support], select(*cols).select_from(joins), **constraints)
|
||||
|
||||
|
||||
def search_supports(**constraints) -> Tuple[List[Output], Optional[int]]:
|
||||
total = None
|
||||
if constraints.pop('include_total', False):
|
||||
total = search_support_count(**constraints)
|
||||
if 'claim_id' in constraints:
|
||||
constraints['claim_hash'] = unhexlify(constraints.pop('claim_id'))[::-1]
|
||||
rows = context().fetchall(select_supports(**constraints))
|
||||
txos = rows_to_txos(rows, include_tx=False)
|
||||
return txos, total
|
||||
|
||||
|
||||
def sum_supports(claim_hash, include_channel_content=False, exclude_own_supports=False) -> Tuple[List[Dict], int]:
|
||||
supporter = Claim.alias("supporter")
|
||||
content = Claim.alias("content")
|
||||
where_condition = (content.c.claim_hash == claim_hash)
|
||||
if include_channel_content:
|
||||
where_condition |= (content.c.channel_hash == claim_hash)
|
||||
support_join_condition = TXO.c.channel_hash == supporter.c.claim_hash
|
||||
if exclude_own_supports:
|
||||
support_join_condition &= TXO.c.channel_hash != claim_hash
|
||||
|
||||
q = select(
|
||||
supporter.c.short_url.label("supporter"),
|
||||
func.sum(TXO.c.amount).label("staked"),
|
||||
).select_from(
|
||||
TXO
|
||||
.join(content, TXO.c.claim_hash == content.c.claim_hash)
|
||||
.join(supporter, support_join_condition)
|
||||
).where(
|
||||
where_condition &
|
||||
(TXO.c.txo_type == TXO_TYPES["support"]) &
|
||||
((TXO.c.address == content.c.address) | ((TXO.c.address != content.c.address) & (TXO.c.spent_height == 0)))
|
||||
).group_by(
|
||||
supporter.c.short_url
|
||||
).order_by(
|
||||
text("staked DESC, supporter ASC")
|
||||
)
|
||||
|
||||
result = context().fetchall(q)
|
||||
total = sum([row['staked'] for row in result])
|
||||
return result, total
|
||||
|
||||
|
||||
def search_support_count(**constraints) -> int:
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
count = context().fetchall(select_supports([func.count().label('total')], **constraints))
|
||||
return count[0]['total'] or 0
|
||||
|
||||
|
||||
channel_claim = Claim.alias('channel')
|
||||
BASE_SELECT_CLAIM_COLUMNS = BASE_SELECT_TXO_COLUMNS + [
|
||||
Claim.c.activation_height,
|
||||
Claim.c.takeover_height,
|
||||
Claim.c.creation_height,
|
||||
Claim.c.expiration_height,
|
||||
Claim.c.is_controlling,
|
||||
Claim.c.channel_hash,
|
||||
Claim.c.reposted_count,
|
||||
Claim.c.reposted_claim_hash,
|
||||
Claim.c.short_url,
|
||||
Claim.c.signed_claim_count,
|
||||
Claim.c.signed_support_count,
|
||||
(Claim.c.amount + Claim.c.staked_support_amount).label('staked_amount'),
|
||||
Claim.c.staked_support_amount,
|
||||
Claim.c.staked_support_count,
|
||||
Claim.c.is_signature_valid,
|
||||
case([(
|
||||
channel_claim.c.short_url.isnot(None),
|
||||
channel_claim.c.short_url + '/' + Claim.c.short_url
|
||||
)]).label('canonical_url'),
|
||||
func.coalesce(Trend.c.trend_local, 0).label('trend_local'),
|
||||
func.coalesce(Trend.c.trend_mixed, 0).label('trend_mixed'),
|
||||
func.coalesce(Trend.c.trend_global, 0).label('trend_global'),
|
||||
func.coalesce(Trend.c.trend_group, 0).label('trend_group'),
|
||||
CensoredClaim.c.censor_type,
|
||||
CensoredClaim.c.censoring_channel_hash
|
||||
]
|
||||
|
||||
|
||||
def select_claims(cols: List = None, for_count=False, **constraints) -> Select:
|
||||
constraints = compat_layer(**constraints)
|
||||
if cols is None:
|
||||
cols = BASE_SELECT_CLAIM_COLUMNS
|
||||
if 'order_by' in constraints:
|
||||
order_by_parts = constraints['order_by']
|
||||
if isinstance(order_by_parts, str):
|
||||
order_by_parts = [order_by_parts]
|
||||
sql_order_by = []
|
||||
for order_by in order_by_parts:
|
||||
is_asc = order_by.startswith('^')
|
||||
column = order_by[1:] if is_asc else order_by
|
||||
if column not in SEARCH_ORDER_FIELDS:
|
||||
raise NameError(f'{column} is not a valid order_by field')
|
||||
if column == 'name':
|
||||
column = 'claim_name'
|
||||
table = "trend" if column.startswith('trend') else "claim"
|
||||
column = f"{table}.{column}"
|
||||
if column in ('trend.trend_group', 'trend.trend_mixed', 'claim.release_time'):
|
||||
column = f"COALESCE({column}, {-1 * (1<<32)})"
|
||||
sql_order_by.append(
|
||||
f"{column} {'ASC' if is_asc else 'DESC'}"
|
||||
)
|
||||
constraints['order_by'] = sql_order_by
|
||||
|
||||
ops = {'<=': '__lte', '>=': '__gte', '<': '__lt', '>': '__gt'}
|
||||
for constraint in SEARCH_INTEGER_PARAMS:
|
||||
if constraint in constraints:
|
||||
value = constraints.pop(constraint)
|
||||
postfix = ''
|
||||
if isinstance(value, str):
|
||||
if len(value) >= 2 and value[:2] in ops:
|
||||
postfix, value = ops[value[:2]], value[2:]
|
||||
elif len(value) >= 1 and value[0] in ops:
|
||||
postfix, value = ops[value[0]], value[1:]
|
||||
if constraint == 'fee_amount':
|
||||
value = Decimal(value)*1000
|
||||
constraints[f'{constraint}{postfix}'] = int(value)
|
||||
|
||||
if 'sequence' in constraints:
|
||||
constraints['order_by'] = 'activation_height ASC'
|
||||
constraints['offset'] = int(constraints.pop('sequence')) - 1
|
||||
constraints['limit'] = 1
|
||||
if 'amount_order' in constraints:
|
||||
constraints['order_by'] = 'staked_amount DESC'
|
||||
constraints['offset'] = int(constraints.pop('amount_order')) - 1
|
||||
constraints['limit'] = 1
|
||||
|
||||
if 'claim_id' in constraints:
|
||||
claim_id = constraints.pop('claim_id')
|
||||
if len(claim_id) == 40:
|
||||
constraints['claim_id'] = claim_id
|
||||
else:
|
||||
constraints['claim_id__like'] = f'{claim_id[:40]}%'
|
||||
elif 'claim_ids' in constraints:
|
||||
constraints['claim_id__in'] = set(constraints.pop('claim_ids'))
|
||||
|
||||
if 'reposted_claim_id' in constraints:
|
||||
constraints['reposted_claim_hash'] = unhexlify(constraints.pop('reposted_claim_id'))[::-1]
|
||||
|
||||
if 'name' in constraints:
|
||||
constraints['normalized'] = normalize_name(constraints.pop('name'))
|
||||
|
||||
if 'public_key_id' in constraints:
|
||||
constraints['public_key_hash'] = (
|
||||
context().ledger.address_to_hash160(constraints.pop('public_key_id')))
|
||||
if 'channel_id' in constraints:
|
||||
channel_id = constraints.pop('channel_id')
|
||||
if channel_id:
|
||||
if isinstance(channel_id, str):
|
||||
channel_id = [channel_id]
|
||||
constraints['channel_hash__in'] = {
|
||||
unhexlify(cid)[::-1] for cid in channel_id
|
||||
}
|
||||
if 'not_channel_id' in constraints:
|
||||
not_channel_ids = constraints.pop('not_channel_id')
|
||||
if not_channel_ids:
|
||||
not_channel_ids_binary = {
|
||||
unhexlify(ncid)[::-1] for ncid in not_channel_ids
|
||||
}
|
||||
constraints['claim_hash__not_in#not_channel_ids'] = not_channel_ids_binary
|
||||
if constraints.get('has_channel_signature', False):
|
||||
constraints['channel_hash__not_in'] = not_channel_ids_binary
|
||||
else:
|
||||
constraints['null_or_not_channel__or'] = {
|
||||
'signature_valid__is_null': True,
|
||||
'channel_hash__not_in': not_channel_ids_binary
|
||||
}
|
||||
if 'is_signature_valid' in constraints:
|
||||
has_channel_signature = constraints.pop('has_channel_signature', False)
|
||||
is_signature_valid = constraints.pop('is_signature_valid')
|
||||
if has_channel_signature:
|
||||
constraints['is_signature_valid'] = is_signature_valid
|
||||
else:
|
||||
constraints['null_or_signature__or'] = {
|
||||
'is_signature_valid__is_null': True,
|
||||
'is_signature_valid': is_signature_valid
|
||||
}
|
||||
elif constraints.pop('has_channel_signature', False):
|
||||
constraints['is_signature_valid__is_not_null'] = True
|
||||
|
||||
if 'txid' in constraints:
|
||||
tx_hash = unhexlify(constraints.pop('txid'))[::-1]
|
||||
nout = constraints.pop('nout', 0)
|
||||
constraints['txo_hash'] = tx_hash + struct.pack('<I', nout)
|
||||
|
||||
if 'claim_type' in constraints:
|
||||
claim_types = constraints.pop('claim_type')
|
||||
if isinstance(claim_types, str):
|
||||
claim_types = {claim_types}
|
||||
if claim_types:
|
||||
constraints['claim_type__in'] = {
|
||||
TXO_TYPES[claim_type] for claim_type in claim_types
|
||||
}
|
||||
if 'stream_type' in constraints:
|
||||
stream_types = constraints.pop('stream_type')
|
||||
if isinstance(stream_types, str):
|
||||
stream_types = {stream_types}
|
||||
if stream_types:
|
||||
constraints['stream_type__in'] = {
|
||||
STREAM_TYPES[stream_type] for stream_type in stream_types
|
||||
}
|
||||
if 'media_type' in constraints:
|
||||
media_types = constraints.pop('media_type')
|
||||
if isinstance(media_types, str):
|
||||
media_types = {media_types}
|
||||
if media_types:
|
||||
constraints['media_type__in'] = set(media_types)
|
||||
|
||||
if 'fee_currency' in constraints:
|
||||
constraints['fee_currency'] = constraints.pop('fee_currency').lower()
|
||||
|
||||
_apply_constraints_for_array_attributes(constraints, 'tag', clean_tags, for_count)
|
||||
_apply_constraints_for_array_attributes(constraints, 'language', lambda _: _, for_count)
|
||||
_apply_constraints_for_array_attributes(constraints, 'location', lambda _: _, for_count)
|
||||
|
||||
if 'text' in constraints:
|
||||
# TODO: fix
|
||||
constraints["search"] = constraints.pop("text")
|
||||
|
||||
return query(
|
||||
[Claim, TXO],
|
||||
select(*cols)
|
||||
.select_from(
|
||||
Claim.join(TXO).join(TX)
|
||||
.join(Trend, Trend.c.claim_hash == Claim.c.claim_hash, isouter=True)
|
||||
.join(channel_claim, Claim.c.channel_hash == channel_claim.c.claim_hash, isouter=True)
|
||||
.join(
|
||||
CensoredClaim,
|
||||
(CensoredClaim.c.claim_hash == Claim.c.claim_hash) |
|
||||
(CensoredClaim.c.claim_hash == Claim.c.reposted_claim_hash) |
|
||||
(CensoredClaim.c.claim_hash == Claim.c.channel_hash),
|
||||
isouter=True
|
||||
)
|
||||
), **constraints
|
||||
)
|
||||
|
||||
|
||||
def protobuf_search_claims(**constraints) -> str:
|
||||
txos, _, censor = search_claims(**constraints)
|
||||
return ResultOutput.to_base64(txos, [], blocked=censor)
|
||||
|
||||
|
||||
def search_claims(**constraints) -> Tuple[List[Output], Optional[int], Optional[Censor]]:
|
||||
ctx = context()
|
||||
search_censor = ctx.get_search_censor()
|
||||
|
||||
total = None
|
||||
if constraints.pop('include_total', False):
|
||||
total = search_claim_count(**constraints)
|
||||
|
||||
constraints['offset'] = abs(constraints.get('offset', 0))
|
||||
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
|
||||
|
||||
channel_url = constraints.pop('channel', None)
|
||||
if channel_url:
|
||||
from .resolve import resolve_url # pylint: disable=import-outside-toplevel
|
||||
channel = resolve_url(channel_url)
|
||||
if isinstance(channel, Output):
|
||||
constraints['channel_hash'] = channel.claim_hash
|
||||
else:
|
||||
return [], total, search_censor
|
||||
|
||||
rows = ctx.fetchall(select_claims(**constraints))
|
||||
rows = search_censor.apply(rows)
|
||||
txos = rows_to_txos(rows, include_tx=False)
|
||||
annotate_with_channels(txos)
|
||||
return txos, total, search_censor
|
||||
|
||||
|
||||
def annotate_with_channels(txos):
|
||||
channel_hashes = set()
|
||||
for txo in txos:
|
||||
if txo.can_decode_claim and txo.claim.is_signed:
|
||||
channel_hashes.add(txo.claim.signing_channel_hash)
|
||||
if channel_hashes:
|
||||
rows = context().fetchall(select_claims(claim_hash__in=channel_hashes))
|
||||
channels = {
|
||||
txo.claim_hash: txo for txo in
|
||||
rows_to_txos(rows, include_tx=False)
|
||||
}
|
||||
for txo in txos:
|
||||
if txo.can_decode_claim and txo.claim.is_signed:
|
||||
txo.channel = channels.get(txo.claim.signing_channel_hash, None)
|
||||
|
||||
|
||||
def search_claim_count(**constraints) -> int:
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
count = context().fetchall(select_claims([func.count().label('total')], **constraints))
|
||||
return count[0]['total'] or 0
|
||||
|
||||
|
||||
CLAIM_HASH_OR_REPOST_HASH_SQL = f"""
|
||||
CASE WHEN claim.claim_type = {TXO_TYPES['repost']}
|
||||
THEN claim.reposted_claim_hash
|
||||
ELSE claim.claim_hash
|
||||
END
|
||||
"""
|
||||
|
||||
|
||||
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
|
||||
any_items = set(cleaner(constraints.pop(f'any_{attr}', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
all_items = set(cleaner(constraints.pop(f'all_{attr}', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
not_items = set(cleaner(constraints.pop(f'not_{attr}', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
|
||||
all_items = {item for item in all_items if item not in not_items}
|
||||
any_items = {item for item in any_items if item not in not_items}
|
||||
|
||||
any_queries = {}
|
||||
|
||||
# if attr == 'tag':
|
||||
# common_tags = any_items & COMMON_TAGS.keys()
|
||||
# if common_tags:
|
||||
# any_items -= common_tags
|
||||
# if len(common_tags) < 5:
|
||||
# for item in common_tags:
|
||||
# index_name = COMMON_TAGS[item]
|
||||
# any_queries[f'#_common_tag_{index_name}'] = f"""
|
||||
# EXISTS(
|
||||
# SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx
|
||||
# WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
|
||||
# AND tag = '{item}'
|
||||
# )
|
||||
# """
|
||||
# elif len(common_tags) >= 5:
|
||||
# constraints.update({
|
||||
# f'$any_common_tag{i}': item for i, item in enumerate(common_tags)
|
||||
# })
|
||||
# values = ', '.join(
|
||||
# f':$any_common_tag{i}' for i in range(len(common_tags))
|
||||
# )
|
||||
# any_queries[f'#_any_common_tags'] = f"""
|
||||
# EXISTS(
|
||||
# SELECT 1 FROM tag WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
|
||||
# AND tag IN ({values})
|
||||
# )
|
||||
# """
|
||||
|
||||
if any_items:
|
||||
|
||||
constraints.update({
|
||||
f'$any_{attr}{i}': item for i, item in enumerate(any_items)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$any_{attr}{i}' for i in range(len(any_items))
|
||||
)
|
||||
if for_count or attr == 'tag':
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
|
||||
if len(any_queries) == 1:
|
||||
constraints.update(any_queries)
|
||||
elif len(any_queries) > 1:
|
||||
constraints[f'ORed_{attr}_queries__any'] = any_queries
|
||||
|
||||
if all_items:
|
||||
constraints[f'$all_{attr}_count'] = len(all_items)
|
||||
constraints.update({
|
||||
f'$all_{attr}{i}': item for i, item in enumerate(all_items)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$all_{attr}{i}' for i in range(len(all_items))
|
||||
)
|
||||
if for_count:
|
||||
constraints[f'#_all_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
GROUP BY claim_hash HAVING COUNT({attr}) = :$all_{attr}_count
|
||||
)
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_all_{attr}'] = f"""
|
||||
{len(all_items)}=(
|
||||
SELECT count(*) FROM {attr} WHERE
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
|
||||
if not_items:
|
||||
constraints.update({
|
||||
f'$not_{attr}{i}': item for i, item in enumerate(not_items)
|
||||
})
|
||||
values = ', '.join(
|
||||
f':$not_{attr}{i}' for i in range(len(not_items))
|
||||
)
|
||||
if for_count:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
NOT EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
@@ -1,676 +0,0 @@
|
||||
import logging
|
||||
from datetime import date
|
||||
from typing import Tuple, List, Optional, Union
|
||||
|
||||
from sqlalchemy import union, func, text, between, distinct, case, false, not_, exists
|
||||
from sqlalchemy.future import select, Select
|
||||
|
||||
from lbry.constants import INVALIDATED_SIGNATURE_GRACE_PERIOD
|
||||
from ...blockchain.transaction import (
|
||||
Transaction, Output, OutputScript, TXRefImmutable
|
||||
)
|
||||
from ..tables import (
|
||||
TX, TXO, TXI, txi_join_account, txo_join_account,
|
||||
Claim, Support, AccountAddress
|
||||
)
|
||||
from ..utils import query, in_account_ids
|
||||
from ..query_context import context
|
||||
from ..constants import TXO_TYPES, CLAIM_TYPE_CODES, MAX_QUERY_VARIABLES
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
minimum_txo_columns = (
|
||||
TXO.c.amount, TXO.c.position.label('txo_position'),
|
||||
TX.c.tx_hash, TX.c.height, TX.c.timestamp,
|
||||
func.substr(TX.c.raw, TXO.c.script_offset + 1, TXO.c.script_length).label('src'),
|
||||
)
|
||||
|
||||
|
||||
def row_to_txo(row):
|
||||
return Output(
|
||||
amount=row.amount,
|
||||
script=OutputScript(row.src),
|
||||
tx_ref=TXRefImmutable.from_hash(row.tx_hash, row.height, row.timestamp),
|
||||
position=row.txo_position,
|
||||
)
|
||||
|
||||
|
||||
def where_txo_type_in(txo_type: Optional[Union[tuple, int]] = None):
|
||||
if txo_type is not None:
|
||||
if isinstance(txo_type, int):
|
||||
return TXO.c.txo_type == txo_type
|
||||
assert len(txo_type) > 0
|
||||
if len(txo_type) == 1:
|
||||
return TXO.c.txo_type == txo_type[0]
|
||||
else:
|
||||
return TXO.c.txo_type.in_(txo_type)
|
||||
return TXO.c.txo_type.in_(CLAIM_TYPE_CODES)
|
||||
|
||||
|
||||
def where_unspent_txos(
|
||||
txo_types: Tuple[int, ...],
|
||||
blocks: Tuple[int, int] = None,
|
||||
missing_in_supports_table: bool = False,
|
||||
missing_in_claims_table: bool = False,
|
||||
missing_or_stale_in_claims_table: bool = False,
|
||||
):
|
||||
condition = where_txo_type_in(txo_types) & (TXO.c.spent_height == 0)
|
||||
if blocks is not None:
|
||||
condition &= between(TXO.c.height, *blocks)
|
||||
if missing_in_supports_table:
|
||||
condition &= not_(
|
||||
exists(select(1).where(Support.c.txo_hash == TXO.c.txo_hash))
|
||||
)
|
||||
elif missing_or_stale_in_claims_table:
|
||||
condition &= not_(
|
||||
exists(select(1).where(Claim.c.txo_hash == TXO.c.txo_hash))
|
||||
)
|
||||
elif missing_in_claims_table:
|
||||
condition &= not_(
|
||||
exists(select(1).where(Claim.c.claim_hash == TXO.c.claim_hash))
|
||||
)
|
||||
return condition
|
||||
|
||||
|
||||
def where_abandoned_claims():
|
||||
return Claim.c.claim_hash.notin_(
|
||||
select(TXO.c.claim_hash).where(where_unspent_txos(CLAIM_TYPE_CODES))
|
||||
)
|
||||
|
||||
|
||||
def count_abandoned_claims():
|
||||
return context().fetchtotal(where_abandoned_claims())
|
||||
|
||||
|
||||
def where_abandoned_supports():
|
||||
return Support.c.txo_hash.notin_(
|
||||
select(TXO.c.txo_hash).where(where_unspent_txos(TXO_TYPES['support']))
|
||||
)
|
||||
|
||||
|
||||
def count_abandoned_supports():
|
||||
return context().fetchtotal(where_abandoned_supports())
|
||||
|
||||
|
||||
def count_unspent_txos(
|
||||
txo_types: Tuple[int, ...],
|
||||
blocks: Tuple[int, int] = None,
|
||||
missing_in_supports_table: bool = False,
|
||||
missing_in_claims_table: bool = False,
|
||||
missing_or_stale_in_claims_table: bool = False,
|
||||
):
|
||||
return context().fetchtotal(
|
||||
where_unspent_txos(
|
||||
txo_types, blocks,
|
||||
missing_in_supports_table,
|
||||
missing_in_claims_table,
|
||||
missing_or_stale_in_claims_table,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def distribute_unspent_txos(
|
||||
txo_types: Tuple[int, ...],
|
||||
blocks: Tuple[int, int] = None,
|
||||
missing_in_supports_table: bool = False,
|
||||
missing_in_claims_table: bool = False,
|
||||
missing_or_stale_in_claims_table: bool = False,
|
||||
number_of_buckets: int = 10
|
||||
) -> Tuple[int, List[Tuple[int, int]]]:
|
||||
chunks = (
|
||||
select(func.ntile(number_of_buckets).over(order_by=TXO.c.height).label('chunk'), TXO.c.height)
|
||||
.where(
|
||||
where_unspent_txos(
|
||||
txo_types, blocks,
|
||||
missing_in_supports_table,
|
||||
missing_in_claims_table,
|
||||
missing_or_stale_in_claims_table,
|
||||
)
|
||||
).cte('chunks')
|
||||
)
|
||||
sql = (
|
||||
select(
|
||||
func.count('*').label('items'),
|
||||
func.min(chunks.c.height).label('start_height'),
|
||||
func.max(chunks.c.height).label('end_height'),
|
||||
).group_by(chunks.c.chunk).order_by(chunks.c.chunk)
|
||||
)
|
||||
total = 0
|
||||
buckets = []
|
||||
for bucket in context().fetchall(sql):
|
||||
total += bucket['items']
|
||||
if len(buckets) > 0:
|
||||
if buckets[-1][-1] == bucket['start_height']:
|
||||
if bucket['start_height'] == bucket['end_height']:
|
||||
continue
|
||||
bucket['start_height'] += 1
|
||||
buckets.append((bucket['start_height'], bucket['end_height']))
|
||||
return total, buckets
|
||||
|
||||
|
||||
def where_changed_support_txos(blocks: Optional[Tuple[int, int]]):
|
||||
return (
|
||||
(TXO.c.txo_type == TXO_TYPES['support']) & (
|
||||
between(TXO.c.height, blocks[0], blocks[-1]) |
|
||||
between(TXO.c.spent_height, blocks[0], blocks[-1])
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def where_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]):
|
||||
return Claim.c.claim_hash.in_(
|
||||
select(TXO.c.claim_hash).where(
|
||||
where_changed_support_txos(blocks)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def count_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]) -> int:
|
||||
sql = (
|
||||
select(func.count(distinct(TXO.c.claim_hash)).label('total'))
|
||||
.where(where_changed_support_txos(blocks))
|
||||
)
|
||||
return context().fetchone(sql)['total']
|
||||
|
||||
|
||||
def where_channels_changed(blocks: Optional[Tuple[int, int]]):
|
||||
channel = TXO.alias('channel')
|
||||
return TXO.c.channel_hash.in_(
|
||||
select(channel.c.claim_hash).where(
|
||||
(channel.c.txo_type == TXO_TYPES['channel']) & (
|
||||
between(channel.c.height, blocks[0], blocks[-1]) |
|
||||
between(channel.c.spent_height, blocks[0], blocks[-1])
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def where_changed_content_txos(blocks: Optional[Tuple[int, int]]):
|
||||
return (
|
||||
(TXO.c.channel_hash.isnot(None)) & (
|
||||
between(TXO.c.height, blocks[0], blocks[-1]) |
|
||||
between(TXO.c.spent_height, blocks[0], blocks[-1])
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def where_channels_with_changed_content(blocks: Optional[Tuple[int, int]]):
|
||||
content = Claim.alias("content")
|
||||
return Claim.c.claim_hash.in_(
|
||||
union(
|
||||
select(TXO.c.channel_hash).where(where_changed_content_txos(blocks)),
|
||||
select(content.c.channel_hash).where(
|
||||
content.c.channel_hash.isnot(None) &
|
||||
# content.c.public_key_height is updated when
|
||||
# channel signature is revalidated
|
||||
between(content.c.public_key_height, blocks[0], blocks[-1])
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def count_channels_with_changed_content(blocks: Optional[Tuple[int, int]]):
|
||||
return context().fetchtotal(where_channels_with_changed_content(blocks))
|
||||
|
||||
|
||||
def where_changed_repost_txos(blocks: Optional[Tuple[int, int]]):
|
||||
return (
|
||||
(TXO.c.txo_type == TXO_TYPES['repost']) & (
|
||||
between(TXO.c.height, blocks[0], blocks[-1]) |
|
||||
between(TXO.c.spent_height, blocks[0], blocks[-1])
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def where_claims_with_changed_reposts(blocks: Optional[Tuple[int, int]]):
|
||||
return Claim.c.claim_hash.in_(
|
||||
select(TXO.c.reposted_claim_hash).where(
|
||||
where_changed_repost_txos(blocks)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def count_claims_with_changed_reposts(blocks: Optional[Tuple[int, int]]):
|
||||
sql = (
|
||||
select(func.count(distinct(TXO.c.reposted_claim_hash)).label('total'))
|
||||
.where(where_changed_repost_txos(blocks))
|
||||
)
|
||||
return context().fetchone(sql)['total']
|
||||
|
||||
|
||||
def where_claims_with_stale_signatures(s, blocks: Optional[Tuple[int, int]], stream=None):
|
||||
stream = Claim.alias('stream') if stream is None else stream
|
||||
channel = Claim.alias('channel')
|
||||
return (
|
||||
s.select_from(stream.join(channel, stream.c.channel_hash == channel.c.claim_hash))
|
||||
.where(
|
||||
(stream.c.public_key_height < channel.c.public_key_height) &
|
||||
(stream.c.public_key_height <= blocks[1]-INVALIDATED_SIGNATURE_GRACE_PERIOD)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def count_claims_with_stale_signatures(blocks: Optional[Tuple[int, int]]):
|
||||
return context().fetchone(
|
||||
where_claims_with_stale_signatures(select(func.count('*').label('total')), blocks)
|
||||
)['total']
|
||||
|
||||
|
||||
def select_transactions(cols, account_ids=None, **constraints):
|
||||
s: Select = select(*cols).select_from(TX)
|
||||
if not {'tx_hash', 'tx_hash__in'}.intersection(constraints):
|
||||
assert account_ids, (
|
||||
"'accounts' argument required when "
|
||||
"no 'tx_hash' constraint is present"
|
||||
)
|
||||
where = in_account_ids(account_ids)
|
||||
tx_hashes = union(
|
||||
select(TXO.c.tx_hash).select_from(txo_join_account).where(where),
|
||||
select(TXI.c.tx_hash).select_from(txi_join_account).where(where)
|
||||
)
|
||||
s = s.where(TX.c.tx_hash.in_(tx_hashes))
|
||||
return context().fetchall(query([TX], s, **constraints))
|
||||
|
||||
|
||||
TXO_NOT_MINE = Output(None, None, is_my_output=False)
|
||||
|
||||
|
||||
def get_raw_transactions(tx_hash__in):
|
||||
return context().fetchall(
|
||||
select(TX.c.tx_hash, TX.c.raw).where(TX.c.tx_hash.in_(tx_hash__in))
|
||||
)
|
||||
|
||||
|
||||
def get_transactions(include_total=False, **constraints) -> Tuple[List[Transaction], Optional[int]]:
|
||||
account_ids = constraints.pop('account_ids', None)
|
||||
include_is_my_input = constraints.pop('include_is_my_input', False)
|
||||
include_is_my_output = constraints.pop('include_is_my_output', False)
|
||||
|
||||
tx_rows = select_transactions(
|
||||
[TX.c.tx_hash, TX.c.raw, TX.c.height, TX.c.position, TX.c.timestamp, TX.c.is_verified],
|
||||
order_by=constraints.pop('order_by', ["height=0 DESC", "height DESC", "position DESC"]),
|
||||
account_ids=account_ids,
|
||||
**constraints
|
||||
)
|
||||
|
||||
txids, txs, txi_txoids = [], [], []
|
||||
for row in tx_rows:
|
||||
txids.append(row['tx_hash'])
|
||||
txs.append(Transaction(
|
||||
raw=row['raw'], height=row['height'], position=row['position'],
|
||||
timestamp=row['timestamp'], is_verified=bool(row['is_verified'])
|
||||
))
|
||||
for txi in txs[-1].inputs:
|
||||
txi_txoids.append(txi.txo_ref.hash)
|
||||
|
||||
annotated_txos = {}
|
||||
for offset in range(0, len(txids), MAX_QUERY_VARIABLES):
|
||||
annotated_txos.update({
|
||||
txo.id: txo for txo in
|
||||
get_txos(
|
||||
wallet_account_ids=account_ids,
|
||||
tx_hash__in=txids[offset:offset + MAX_QUERY_VARIABLES], order_by='txo.tx_hash',
|
||||
include_is_my_input=include_is_my_input,
|
||||
include_is_my_output=include_is_my_output,
|
||||
)[0]
|
||||
})
|
||||
|
||||
referenced_txos = {}
|
||||
for offset in range(0, len(txi_txoids), MAX_QUERY_VARIABLES):
|
||||
referenced_txos.update({
|
||||
txo.id: txo for txo in
|
||||
get_txos(
|
||||
wallet_account_ids=account_ids,
|
||||
txo_hash__in=txi_txoids[offset:offset + MAX_QUERY_VARIABLES], order_by='txo.txo_hash',
|
||||
include_is_my_output=include_is_my_output,
|
||||
)[0]
|
||||
})
|
||||
|
||||
for tx in txs:
|
||||
for txi in tx.inputs:
|
||||
txo = referenced_txos.get(txi.txo_ref.id)
|
||||
if txo:
|
||||
txi.txo_ref = txo.ref
|
||||
for txo in tx.outputs:
|
||||
_txo = annotated_txos.get(txo.id)
|
||||
if _txo:
|
||||
txo.update_annotations(_txo)
|
||||
else:
|
||||
txo.update_annotations(TXO_NOT_MINE)
|
||||
|
||||
for tx in txs:
|
||||
txos = tx.outputs
|
||||
if len(txos) >= 2 and txos[1].can_decode_purchase_data:
|
||||
txos[0].purchase = txos[1]
|
||||
|
||||
return txs, get_transaction_count(**constraints) if include_total else None
|
||||
|
||||
|
||||
def get_transaction_count(**constraints):
|
||||
constraints.pop('wallet', None)
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
count = select_transactions([func.count().label('total')], **constraints)
|
||||
return count[0]['total'] or 0
|
||||
|
||||
|
||||
BASE_SELECT_TXO_COLUMNS = [
|
||||
TX.c.tx_hash, TX.c.raw, TX.c.height, TX.c.position.label('tx_position'),
|
||||
TX.c.is_verified, TX.c.timestamp,
|
||||
TXO.c.txo_type, TXO.c.position.label('txo_position'), TXO.c.amount, TXO.c.spent_height,
|
||||
TXO.c.script_offset, TXO.c.script_length,
|
||||
]
|
||||
|
||||
|
||||
def select_txos(
|
||||
cols=None, account_ids=None, is_my_input=None,
|
||||
is_my_output=True, is_my_input_or_output=None, exclude_internal_transfers=False,
|
||||
include_is_my_input=False, claim_id_not_in_claim_table=None,
|
||||
txo_id_not_in_claim_table=None, txo_id_not_in_support_table=None,
|
||||
**constraints
|
||||
) -> Select:
|
||||
if cols is None:
|
||||
cols = BASE_SELECT_TXO_COLUMNS
|
||||
s: Select = select(*cols)
|
||||
if account_ids:
|
||||
my_addresses = select(AccountAddress.c.address).where(in_account_ids(account_ids))
|
||||
if is_my_input_or_output:
|
||||
include_is_my_input = True
|
||||
s = s.where(
|
||||
TXO.c.address.in_(my_addresses) | (
|
||||
(TXI.c.address.isnot(None)) &
|
||||
(TXI.c.address.in_(my_addresses))
|
||||
)
|
||||
)
|
||||
else:
|
||||
if is_my_output:
|
||||
s = s.where(TXO.c.address.in_(my_addresses))
|
||||
elif is_my_output is False:
|
||||
s = s.where(TXO.c.address.notin_(my_addresses))
|
||||
if is_my_input:
|
||||
include_is_my_input = True
|
||||
s = s.where(
|
||||
(TXI.c.address.isnot(None)) &
|
||||
(TXI.c.address.in_(my_addresses))
|
||||
)
|
||||
elif is_my_input is False:
|
||||
include_is_my_input = True
|
||||
s = s.where(
|
||||
(TXI.c.address.is_(None)) |
|
||||
(TXI.c.address.notin_(my_addresses))
|
||||
)
|
||||
if exclude_internal_transfers:
|
||||
include_is_my_input = True
|
||||
s = s.where(
|
||||
(TXO.c.txo_type != TXO_TYPES['other']) |
|
||||
(TXO.c.address.notin_(my_addresses))
|
||||
(TXI.c.address.is_(None)) |
|
||||
(TXI.c.address.notin_(my_addresses))
|
||||
)
|
||||
joins = TXO.join(TX)
|
||||
if constraints.pop('is_spent', None) is False:
|
||||
s = s.where((TXO.c.spent_height == 0) & (TXO.c.is_reserved == false()))
|
||||
if include_is_my_input:
|
||||
joins = joins.join(TXI, (TXI.c.position == 0) & (TXI.c.tx_hash == TXO.c.tx_hash), isouter=True)
|
||||
if claim_id_not_in_claim_table:
|
||||
s = s.where(TXO.c.claim_hash.notin_(select(Claim.c.claim_hash)))
|
||||
elif txo_id_not_in_claim_table:
|
||||
s = s.where(TXO.c.txo_hash.notin_(select(Claim.c.txo_hash)))
|
||||
elif txo_id_not_in_support_table:
|
||||
s = s.where(TXO.c.txo_hash.notin_(select(Support.c.txo_hash)))
|
||||
return query([TXO, TX], s.select_from(joins), **constraints)
|
||||
|
||||
|
||||
META_ATTRS = (
|
||||
'activation_height', 'takeover_height', 'creation_height', 'staked_amount',
|
||||
'short_url', 'canonical_url', 'staked_support_amount', 'staked_support_count',
|
||||
'signed_claim_count', 'signed_support_count', 'is_signature_valid',
|
||||
'trend_group', 'trend_mixed', 'trend_local', 'trend_global',
|
||||
'reposted_count', 'expiration_height',
|
||||
)
|
||||
|
||||
|
||||
def rows_to_txos(rows: List[dict], include_tx=True) -> List[Output]:
|
||||
txos = []
|
||||
tx_cache = {}
|
||||
for row in rows:
|
||||
if include_tx:
|
||||
if row['tx_hash'] not in tx_cache:
|
||||
tx_cache[row['tx_hash']] = Transaction(
|
||||
row['raw'], height=row['height'], position=row['tx_position'],
|
||||
timestamp=row['timestamp'],
|
||||
is_verified=bool(row['is_verified']),
|
||||
)
|
||||
txo = tx_cache[row['tx_hash']].outputs[row['txo_position']]
|
||||
else:
|
||||
source = row['raw'][row['script_offset']:row['script_offset']+row['script_length']]
|
||||
txo = Output(
|
||||
amount=row['amount'],
|
||||
script=OutputScript(source),
|
||||
tx_ref=TXRefImmutable.from_hash(row['tx_hash'], row['height'], row['timestamp']),
|
||||
position=row['txo_position'],
|
||||
)
|
||||
txo.spent_height = row['spent_height']
|
||||
if 'is_my_input' in row:
|
||||
txo.is_my_input = bool(row['is_my_input'])
|
||||
if 'is_my_output' in row:
|
||||
txo.is_my_output = bool(row['is_my_output'])
|
||||
if 'is_my_input' in row and 'is_my_output' in row:
|
||||
if txo.is_my_input and txo.is_my_output and row['txo_type'] == TXO_TYPES['other']:
|
||||
txo.is_internal_transfer = True
|
||||
else:
|
||||
txo.is_internal_transfer = False
|
||||
if 'received_tips' in row:
|
||||
txo.received_tips = row['received_tips']
|
||||
for attr in META_ATTRS:
|
||||
if attr in row:
|
||||
txo.meta[attr] = row[attr]
|
||||
txos.append(txo)
|
||||
return txos
|
||||
|
||||
|
||||
def get_txos(no_tx=False, include_total=False, **constraints) -> Tuple[List[Output], Optional[int]]:
|
||||
wallet_account_ids = constraints.pop('wallet_account_ids', [])
|
||||
include_is_my_input = constraints.get('include_is_my_input', False)
|
||||
include_is_my_output = constraints.pop('include_is_my_output', False)
|
||||
include_received_tips = constraints.pop('include_received_tips', False)
|
||||
|
||||
select_columns = BASE_SELECT_TXO_COLUMNS + [
|
||||
TXO.c.claim_name
|
||||
]
|
||||
|
||||
my_accounts = None
|
||||
if wallet_account_ids:
|
||||
my_accounts = select(AccountAddress.c.address).where(in_account_ids(wallet_account_ids))
|
||||
|
||||
if include_is_my_output and my_accounts is not None:
|
||||
if constraints.get('is_my_output', None) in (True, False):
|
||||
select_columns.append(text(f"{1 if constraints['is_my_output'] else 0} AS is_my_output"))
|
||||
else:
|
||||
select_columns.append(TXO.c.address.in_(my_accounts).label('is_my_output'))
|
||||
|
||||
if include_is_my_input and my_accounts is not None:
|
||||
if constraints.get('is_my_input', None) in (True, False):
|
||||
select_columns.append(text(f"{1 if constraints['is_my_input'] else 0} AS is_my_input"))
|
||||
else:
|
||||
select_columns.append((
|
||||
(TXI.c.address.isnot(None)) &
|
||||
(TXI.c.address.in_(my_accounts))
|
||||
).label('is_my_input'))
|
||||
|
||||
if include_received_tips:
|
||||
support = TXO.alias('support')
|
||||
select_columns.append(
|
||||
select(func.coalesce(func.sum(support.c.amount), 0))
|
||||
.select_from(support).where(
|
||||
(support.c.claim_hash == TXO.c.claim_hash) &
|
||||
(support.c.txo_type == TXO_TYPES['support']) &
|
||||
(support.c.address.in_(my_accounts)) &
|
||||
(support.c.txo_hash.notin_(select(TXI.c.txo_hash)))
|
||||
).label('received_tips')
|
||||
)
|
||||
|
||||
if 'order_by' not in constraints or constraints['order_by'] == 'height':
|
||||
constraints['order_by'] = [
|
||||
"tx.height=0 DESC", "tx.height DESC", "tx.position DESC", "txo.position"
|
||||
]
|
||||
elif constraints.get('order_by', None) == 'none':
|
||||
del constraints['order_by']
|
||||
|
||||
rows = context().fetchall(select_txos(select_columns, **constraints))
|
||||
txos = rows_to_txos(rows, not no_tx)
|
||||
|
||||
channel_hashes = set()
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.can_decode_claim:
|
||||
if txo.claim.is_signed:
|
||||
channel_hashes.add(txo.claim.signing_channel_hash)
|
||||
|
||||
if channel_hashes:
|
||||
channels = {
|
||||
txo.claim_hash: txo for txo in
|
||||
get_txos(
|
||||
txo_type=TXO_TYPES['channel'], spent_height=0,
|
||||
wallet_account_ids=wallet_account_ids, claim_hash__in=channel_hashes
|
||||
)[0]
|
||||
}
|
||||
for txo in txos:
|
||||
if txo.is_claim and txo.can_decode_claim:
|
||||
txo.channel = channels.get(txo.claim.signing_channel_hash, None)
|
||||
|
||||
return txos, get_txo_count(**constraints) if include_total else None
|
||||
|
||||
|
||||
def _clean_txo_constraints_for_aggregation(constraints):
|
||||
constraints.pop('include_is_my_input', None)
|
||||
constraints.pop('include_is_my_output', None)
|
||||
constraints.pop('include_received_tips', None)
|
||||
constraints.pop('wallet_account_ids', None)
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
|
||||
|
||||
def get_txo_count(**constraints):
|
||||
_clean_txo_constraints_for_aggregation(constraints)
|
||||
count = context().fetchall(select_txos([func.count().label('total')], **constraints))
|
||||
return count[0]['total'] or 0
|
||||
|
||||
|
||||
def get_txo_sum(**constraints):
|
||||
_clean_txo_constraints_for_aggregation(constraints)
|
||||
result = context().fetchall(select_txos([func.sum(TXO.c.amount).label('total')], **constraints))
|
||||
return result[0]['total'] or 0
|
||||
|
||||
|
||||
def get_balance(account_ids):
|
||||
ctx = context()
|
||||
my_addresses = select(AccountAddress.c.address).where(in_account_ids(account_ids))
|
||||
if ctx.is_postgres:
|
||||
txo_address_check = TXO.c.address == func.any(func.array(my_addresses))
|
||||
txi_address_check = TXI.c.address == func.any(func.array(my_addresses))
|
||||
else:
|
||||
txo_address_check = TXO.c.address.in_(my_addresses)
|
||||
txi_address_check = TXI.c.address.in_(my_addresses)
|
||||
s: Select = (
|
||||
select(
|
||||
func.coalesce(func.sum(TXO.c.amount), 0).label("total"),
|
||||
func.coalesce(func.sum(case(
|
||||
[(TXO.c.txo_type != TXO_TYPES["other"], TXO.c.amount)],
|
||||
)), 0).label("reserved"),
|
||||
func.coalesce(func.sum(case(
|
||||
[(where_txo_type_in(CLAIM_TYPE_CODES), TXO.c.amount)],
|
||||
)), 0).label("claims"),
|
||||
func.coalesce(func.sum(case(
|
||||
[(where_txo_type_in(TXO_TYPES["support"]), TXO.c.amount)],
|
||||
)), 0).label("supports"),
|
||||
func.coalesce(func.sum(case(
|
||||
[(where_txo_type_in(TXO_TYPES["support"]) & (
|
||||
(TXI.c.address.isnot(None)) & txi_address_check
|
||||
), TXO.c.amount)],
|
||||
)), 0).label("my_supports"),
|
||||
)
|
||||
.where((TXO.c.spent_height == 0) & txo_address_check)
|
||||
.select_from(
|
||||
TXO.join(TXI, (TXI.c.position == 0) & (TXI.c.tx_hash == TXO.c.tx_hash), isouter=True)
|
||||
)
|
||||
)
|
||||
result = ctx.fetchone(s)
|
||||
return {
|
||||
"total": result["total"],
|
||||
"available": result["total"] - result["reserved"],
|
||||
"reserved": result["reserved"],
|
||||
"reserved_subtotals": {
|
||||
"claims": result["claims"],
|
||||
"supports": result["my_supports"],
|
||||
"tips": result["supports"] - result["my_supports"]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def get_report(account_ids):
|
||||
return
|
||||
|
||||
|
||||
def get_txo_plot(start_day=None, days_back=0, end_day=None, days_after=None, **constraints):
|
||||
_clean_txo_constraints_for_aggregation(constraints)
|
||||
if start_day is None:
|
||||
# TODO: Fix
|
||||
current_ordinal = 0 # self.ledger.headers.estimated_date(self.ledger.headers.height).toordinal()
|
||||
constraints['day__gte'] = current_ordinal - days_back
|
||||
else:
|
||||
constraints['day__gte'] = date.fromisoformat(start_day).toordinal()
|
||||
if end_day is not None:
|
||||
constraints['day__lte'] = date.fromisoformat(end_day).toordinal()
|
||||
elif days_after is not None:
|
||||
constraints['day__lte'] = constraints['day__gte'] + days_after
|
||||
plot = context().fetchall(select_txos(
|
||||
[TX.c.day, func.sum(TXO.c.amount).label('total')],
|
||||
group_by='day', order_by='day', **constraints
|
||||
))
|
||||
for row in plot:
|
||||
row['day'] = date.fromordinal(row['day'])
|
||||
return plot
|
||||
|
||||
|
||||
def get_purchases(**constraints) -> Tuple[List[Output], Optional[int]]:
|
||||
accounts = constraints.pop('accounts', None)
|
||||
assert accounts, "'accounts' argument required to find purchases"
|
||||
if not {'purchased_claim_hash', 'purchased_claim_hash__in'}.intersection(constraints):
|
||||
constraints['purchased_claim_hash__is_not_null'] = True
|
||||
constraints['tx_hash__in'] = (
|
||||
select(TXI.c.tx_hash).select_from(txi_join_account).where(in_account_ids(accounts))
|
||||
)
|
||||
txs, count = get_transactions(**constraints)
|
||||
return [tx.outputs[0] for tx in txs], count
|
||||
|
||||
|
||||
def get_supports_summary(self, **constraints):
|
||||
return get_txos(
|
||||
txo_type=TXO_TYPES['support'],
|
||||
spent_height=0, is_my_output=True,
|
||||
include_is_my_input=True,
|
||||
no_tx=True,
|
||||
**constraints
|
||||
)
|
||||
|
||||
|
||||
def reserve_outputs(txo_hashes, is_reserved=True):
|
||||
context().execute(
|
||||
TXO.update()
|
||||
.values(is_reserved=is_reserved)
|
||||
.where(TXO.c.txo_hash.in_(txo_hashes))
|
||||
)
|
||||
|
||||
|
||||
def release_all_outputs(account_id):
|
||||
context().execute(
|
||||
TXO.update().values(is_reserved=False).where(
|
||||
TXO.c.is_reserved & TXO.c.address.in_(
|
||||
select(AccountAddress.c.address).where(in_account_ids(account_id))
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -1,24 +0,0 @@
|
||||
from sqlalchemy import exists
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from ..query_context import context
|
||||
from ..tables import Wallet
|
||||
|
||||
|
||||
def has_wallet(wallet_id: str) -> bool:
|
||||
sql = select(exists(select(Wallet.c.wallet_id).where(Wallet.c.wallet_id == wallet_id)))
|
||||
return context().execute(sql).fetchone()[0]
|
||||
|
||||
|
||||
def get_wallet(wallet_id: str):
|
||||
return context().fetchone(
|
||||
select(Wallet.c.data).where(Wallet.c.wallet_id == wallet_id)
|
||||
)
|
||||
|
||||
|
||||
def add_wallet(wallet_id: str, data: str):
|
||||
c = context()
|
||||
c.execute(
|
||||
c.insert_or_replace(Wallet, ["data"])
|
||||
.values(wallet_id=wallet_id, data=data)
|
||||
)
|
||||
@@ -1,766 +0,0 @@
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
import functools
|
||||
from io import BytesIO
|
||||
import multiprocessing as mp
|
||||
from decimal import Decimal
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from dataclasses import dataclass, field
|
||||
from contextvars import ContextVar
|
||||
|
||||
from sqlalchemy import create_engine, inspect, bindparam, func, case, exists, event as sqlalchemy_event
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.sql import Insert, text
|
||||
try:
|
||||
from pgcopy import CopyManager
|
||||
except ImportError:
|
||||
CopyManager = None
|
||||
|
||||
from lbry.event import EventQueuePublisher
|
||||
from lbry.blockchain.ledger import Ledger
|
||||
from lbry.blockchain.transaction import Transaction, Output, Input
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.result import Censor
|
||||
from lbry.schema.mime_types import guess_stream_type
|
||||
|
||||
from .utils import pg_insert
|
||||
from .tables import (
|
||||
Block, BlockFilter,
|
||||
TX, TXFilter, TXO, TXI, Claim, Tag, Support
|
||||
)
|
||||
from .constants import TXO_TYPES, STREAM_TYPES
|
||||
|
||||
|
||||
_context: ContextVar['QueryContext'] = ContextVar('_context')
|
||||
|
||||
|
||||
@dataclass
|
||||
class QueryContext:
|
||||
engine: Engine
|
||||
ledger: Ledger
|
||||
message_queue: mp.Queue
|
||||
stop_event: mp.Event
|
||||
stack: List[List]
|
||||
metrics: Dict
|
||||
is_tracking_metrics: bool
|
||||
blocked_streams: Dict
|
||||
blocked_channels: Dict
|
||||
filtered_streams: Dict
|
||||
filtered_channels: Dict
|
||||
pid: int
|
||||
|
||||
# QueryContext __enter__/__exit__ state
|
||||
current_timer_name: Optional[str] = None
|
||||
current_timer_time: float = 0
|
||||
current_progress: Optional['ProgressContext'] = None
|
||||
|
||||
copy_managers: Dict[str, CopyManager] = field(default_factory=dict)
|
||||
_variable_limit: Optional[int] = None
|
||||
|
||||
@property
|
||||
def is_postgres(self):
|
||||
return self.engine.dialect.name == 'postgresql'
|
||||
|
||||
@property
|
||||
def is_sqlite(self):
|
||||
return self.engine.dialect.name == 'sqlite'
|
||||
|
||||
@property
|
||||
def variable_limit(self):
|
||||
if self._variable_limit is not None:
|
||||
return self._variable_limit
|
||||
if self.is_sqlite:
|
||||
for result in self.fetchall(text('PRAGMA COMPILE_OPTIONS;')):
|
||||
for _, value in result.items():
|
||||
if value.startswith('MAX_VARIABLE_NUMBER'):
|
||||
self._variable_limit = int(value.split('=')[1])
|
||||
return self._variable_limit
|
||||
self._variable_limit = 999 # todo: default for 3.32.0 is 32766, but we are still hitting 999 somehow
|
||||
else:
|
||||
self._variable_limit = 32766
|
||||
return self._variable_limit
|
||||
|
||||
def raise_unsupported_dialect(self):
|
||||
raise RuntimeError(f'Unsupported database dialect: {self.engine.dialect.name}.')
|
||||
|
||||
@classmethod
|
||||
def get_resolve_censor(cls) -> Censor:
|
||||
return Censor(Censor.RESOLVE)
|
||||
|
||||
@classmethod
|
||||
def get_search_censor(cls) -> Censor:
|
||||
return Censor(Censor.SEARCH)
|
||||
|
||||
def pg_copy(self, table, rows):
|
||||
with self.engine.begin() as c:
|
||||
copy_manager = self.copy_managers.get(table.name)
|
||||
if copy_manager is None:
|
||||
self.copy_managers[table.name] = copy_manager = CopyManager(
|
||||
c.connection, table.name, rows[0].keys()
|
||||
)
|
||||
copy_manager.conn = c.connection
|
||||
copy_manager.copy(map(dict.values, rows), BytesIO)
|
||||
copy_manager.conn = None
|
||||
|
||||
def pg_has_pk_constraint(self, table_name):
|
||||
claim_constraints = self.fetchall(text(
|
||||
f"select * from information_schema.table_constraints as tc "
|
||||
f"where tc.table_name='{table_name}' and constraint_type='PRIMARY KEY'"
|
||||
))
|
||||
return len(claim_constraints) > 0
|
||||
|
||||
def connect_without_transaction(self):
|
||||
return self.engine.connect().execution_options(isolation_level="AUTOCOMMIT")
|
||||
|
||||
def connect_streaming(self):
|
||||
return self.engine.connect().execution_options(stream_results=True)
|
||||
|
||||
def execute_notx(self, sql, *args):
|
||||
with self.connect_without_transaction() as c:
|
||||
return c.execute(sql, *args)
|
||||
|
||||
def execute(self, sql, *args):
|
||||
with self.engine.begin() as c:
|
||||
return c.execute(sql, *args)
|
||||
|
||||
def fetchone(self, sql, *args):
|
||||
with self.engine.begin() as c:
|
||||
row = c.execute(sql, *args).fetchone()
|
||||
return dict(row._mapping) if row else row
|
||||
|
||||
def fetchall(self, sql, *args):
|
||||
with self.engine.begin() as c:
|
||||
rows = c.execute(sql, *args).fetchall()
|
||||
return [dict(row._mapping) for row in rows]
|
||||
|
||||
def fetchtotal(self, condition) -> int:
|
||||
sql = select(func.count('*').label('total')).where(condition)
|
||||
return self.fetchone(sql)['total']
|
||||
|
||||
def fetchmax(self, column, default: int) -> int:
|
||||
sql = select(func.coalesce(func.max(column), default).label('max_result'))
|
||||
return self.fetchone(sql)['max_result']
|
||||
|
||||
def has_records(self, table) -> bool:
|
||||
sql = select(exists([1], from_obj=table).label('result'))
|
||||
return bool(self.fetchone(sql)['result'])
|
||||
|
||||
def insert_or_ignore(self, table):
|
||||
if self.is_sqlite:
|
||||
return table.insert().prefix_with("OR IGNORE")
|
||||
elif self.is_postgres:
|
||||
return pg_insert(table).on_conflict_do_nothing()
|
||||
else:
|
||||
self.raise_unsupported_dialect()
|
||||
|
||||
def insert_or_replace(self, table, replace):
|
||||
if self.is_sqlite:
|
||||
return table.insert().prefix_with("OR REPLACE")
|
||||
elif self.is_postgres:
|
||||
insert = pg_insert(table)
|
||||
return insert.on_conflict_do_update(
|
||||
table.primary_key, set_={col: getattr(insert.excluded, col) for col in replace}
|
||||
)
|
||||
else:
|
||||
self.raise_unsupported_dialect()
|
||||
|
||||
def has_table(self, table):
|
||||
return inspect(self.engine).has_table(table)
|
||||
|
||||
def get_bulk_loader(self) -> 'BulkLoader':
|
||||
return BulkLoader(self)
|
||||
|
||||
def reset_metrics(self):
|
||||
self.stack = []
|
||||
self.metrics = {}
|
||||
|
||||
def with_timer(self, timer_name: str) -> 'QueryContext':
|
||||
self.current_timer_name = timer_name
|
||||
return self
|
||||
|
||||
@property
|
||||
def elapsed(self):
|
||||
return time.perf_counter() - self.current_timer_time
|
||||
|
||||
def __enter__(self) -> 'QueryContext':
|
||||
self.current_timer_time = time.perf_counter()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.current_timer_name = None
|
||||
self.current_timer_time = 0
|
||||
self.current_progress = None
|
||||
|
||||
|
||||
def context(with_timer: str = None) -> 'QueryContext':
|
||||
if isinstance(with_timer, str):
|
||||
return _context.get().with_timer(with_timer)
|
||||
return _context.get()
|
||||
|
||||
|
||||
def set_postgres_settings(connection, _):
|
||||
cursor = connection.cursor()
|
||||
cursor.execute('SET work_mem="500MB";')
|
||||
cursor.execute('COMMIT;')
|
||||
cursor.close()
|
||||
|
||||
|
||||
def set_sqlite_settings(connection, _):
|
||||
connection.isolation_level = None
|
||||
cursor = connection.cursor()
|
||||
cursor.execute('PRAGMA journal_mode=WAL;')
|
||||
cursor.close()
|
||||
|
||||
|
||||
def do_sqlite_begin(connection):
|
||||
# see: https://bit.ly/3j4vvXm
|
||||
connection.exec_driver_sql("BEGIN")
|
||||
|
||||
|
||||
def initialize(
|
||||
ledger: Ledger, message_queue: mp.Queue, stop_event: mp.Event,
|
||||
track_metrics=False, block_and_filter=None):
|
||||
url = ledger.conf.db_url_or_default
|
||||
engine = create_engine(url)
|
||||
if engine.name == "postgresql":
|
||||
sqlalchemy_event.listen(engine, "connect", set_postgres_settings)
|
||||
elif engine.name == "sqlite":
|
||||
sqlalchemy_event.listen(engine, "connect", set_sqlite_settings)
|
||||
sqlalchemy_event.listen(engine, "begin", do_sqlite_begin)
|
||||
if block_and_filter is not None:
|
||||
blocked_streams, blocked_channels, filtered_streams, filtered_channels = block_and_filter
|
||||
else:
|
||||
blocked_streams = blocked_channels = filtered_streams = filtered_channels = {}
|
||||
_context.set(
|
||||
QueryContext(
|
||||
pid=os.getpid(), engine=engine,
|
||||
ledger=ledger, message_queue=message_queue, stop_event=stop_event,
|
||||
stack=[], metrics={}, is_tracking_metrics=track_metrics,
|
||||
blocked_streams=blocked_streams, blocked_channels=blocked_channels,
|
||||
filtered_streams=filtered_streams, filtered_channels=filtered_channels,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def uninitialize():
|
||||
ctx = _context.get(None)
|
||||
if ctx is not None:
|
||||
ctx.engine.dispose()
|
||||
_context.set(None)
|
||||
|
||||
|
||||
class Event:
|
||||
_events: List['Event'] = []
|
||||
__slots__ = 'id', 'name', 'units'
|
||||
|
||||
def __init__(self, name: str, units: Tuple[str]):
|
||||
self.id = None
|
||||
self.name = name
|
||||
self.units = units
|
||||
|
||||
@classmethod
|
||||
def get_by_id(cls, event_id) -> 'Event':
|
||||
return cls._events[event_id]
|
||||
|
||||
@classmethod
|
||||
def get_by_name(cls, name) -> 'Event':
|
||||
for event in cls._events:
|
||||
if event.name == name:
|
||||
return event
|
||||
|
||||
@classmethod
|
||||
def add(cls, name: str, *units: str) -> 'Event':
|
||||
assert cls.get_by_name(name) is None, f"Event {name} already exists."
|
||||
assert name.count('.') == 3, f"Event {name} does not follow pattern of: [module].sync.[phase].[task]"
|
||||
event = cls(name, units)
|
||||
cls._events.append(event)
|
||||
event.id = cls._events.index(event)
|
||||
return event
|
||||
|
||||
|
||||
def event_emitter(name: str, *units: str, throttle=1):
|
||||
event = Event.add(name, *units)
|
||||
|
||||
def wrapper(f):
|
||||
@functools.wraps(f)
|
||||
def with_progress(*args, **kwargs):
|
||||
with progress(event, throttle=throttle) as p:
|
||||
try:
|
||||
return f(*args, **kwargs, p=p)
|
||||
except BreakProgress:
|
||||
raise
|
||||
except:
|
||||
traceback.print_exc()
|
||||
raise
|
||||
return with_progress
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class ProgressPublisher(EventQueuePublisher):
|
||||
|
||||
def message_to_event(self, message):
|
||||
total, extra = None, None
|
||||
if len(message) == 3:
|
||||
event_id, progress_id, done = message
|
||||
elif len(message) == 5:
|
||||
event_id, progress_id, done, total, extra = message
|
||||
else:
|
||||
raise TypeError("progress message must be tuple of 3 or 5 values.")
|
||||
event = Event.get_by_id(event_id)
|
||||
d = {
|
||||
"event": event.name,
|
||||
"data": {"id": progress_id, "done": done}
|
||||
}
|
||||
if total is not None:
|
||||
d['data']['total'] = total
|
||||
d['data']['units'] = event.units
|
||||
if isinstance(extra, dict):
|
||||
d['data'].update(extra)
|
||||
return d
|
||||
|
||||
|
||||
class BreakProgress(Exception):
|
||||
"""Break out of progress when total is 0."""
|
||||
|
||||
|
||||
class Progress:
|
||||
|
||||
def __init__(self, message_queue: mp.Queue, event: Event, throttle=1):
|
||||
self.message_queue = message_queue
|
||||
self.event = event
|
||||
self.progress_id = 0
|
||||
self.throttle = throttle
|
||||
self.last_done = (0,)*len(event.units)
|
||||
self.last_done_queued = (0,)*len(event.units)
|
||||
self.totals = (0,)*len(event.units)
|
||||
|
||||
def __enter__(self) -> 'Progress':
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self.last_done != self.last_done_queued:
|
||||
self.message_queue.put((self.event.id, self.progress_id, self.last_done))
|
||||
self.last_done_queued = self.last_done
|
||||
if exc_type == BreakProgress:
|
||||
return True
|
||||
if self.last_done != self.totals: # or exc_type is not None:
|
||||
# TODO: add exception info into closing message if there is any
|
||||
self.message_queue.put((
|
||||
self.event.id, self.progress_id, (-1,)*len(self.event.units)
|
||||
))
|
||||
|
||||
def start(self, *totals: int, progress_id=0, label=None, extra=None):
|
||||
assert len(totals) == len(self.event.units), \
|
||||
f"Totals {totals} do not match up with units {self.event.units}."
|
||||
if not any(totals):
|
||||
raise BreakProgress
|
||||
self.totals = totals
|
||||
self.progress_id = progress_id
|
||||
extra = {} if extra is None else extra.copy()
|
||||
if label is not None:
|
||||
extra['label'] = label
|
||||
self.step(*((0,)*len(totals)), force=True, extra=extra)
|
||||
|
||||
def step(self, *done: int, force=False, extra=None):
|
||||
if done == ():
|
||||
assert len(self.totals) == 1, "Incrementing step() only works with one unit progress."
|
||||
done = (self.last_done[0]+1,)
|
||||
assert len(done) == len(self.totals), \
|
||||
f"Done elements {done} don't match total elements {self.totals}."
|
||||
self.last_done = done
|
||||
send_condition = force or extra is not None or (
|
||||
# throttle rate of events being generated (only throttles first unit value)
|
||||
(self.throttle == 1 or done[0] % self.throttle == 0) and
|
||||
# deduplicate finish event by not sending a step where done == total
|
||||
any(i < j for i, j in zip(done, self.totals)) and
|
||||
# deduplicate same event
|
||||
done != self.last_done_queued
|
||||
)
|
||||
if send_condition:
|
||||
if extra is not None:
|
||||
self.message_queue.put_nowait(
|
||||
(self.event.id, self.progress_id, done, self.totals, extra)
|
||||
)
|
||||
else:
|
||||
self.message_queue.put_nowait(
|
||||
(self.event.id, self.progress_id, done)
|
||||
)
|
||||
self.last_done_queued = done
|
||||
|
||||
def add(self, *done: int, force=False, extra=None):
|
||||
assert len(done) == len(self.last_done), \
|
||||
f"Done elements {done} don't match total elements {self.last_done}."
|
||||
self.step(
|
||||
*(i+j for i, j in zip(self.last_done, done)),
|
||||
force=force, extra=extra
|
||||
)
|
||||
|
||||
def iter(self, items: List):
|
||||
self.start(len(items))
|
||||
for item in items:
|
||||
yield item
|
||||
self.step()
|
||||
|
||||
|
||||
class ProgressContext(Progress):
|
||||
|
||||
def __init__(self, ctx: QueryContext, event: Event, throttle=1):
|
||||
super().__init__(ctx.message_queue, event, throttle)
|
||||
self.ctx = ctx
|
||||
|
||||
def __enter__(self) -> 'ProgressContext':
|
||||
self.ctx.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
return any((
|
||||
self.ctx.__exit__(exc_type, exc_val, exc_tb),
|
||||
super().__exit__(exc_type, exc_val, exc_tb)
|
||||
))
|
||||
|
||||
|
||||
def progress(e: Event, throttle=1) -> ProgressContext:
|
||||
ctx = context(e.name)
|
||||
ctx.current_progress = ProgressContext(ctx, e, throttle=throttle)
|
||||
return ctx.current_progress
|
||||
|
||||
|
||||
class BulkLoader:
|
||||
|
||||
def __init__(self, ctx: QueryContext):
|
||||
self.ctx = ctx
|
||||
self.ledger = ctx.ledger
|
||||
self.blocks = []
|
||||
self.txs = []
|
||||
self.txos = []
|
||||
self.txis = []
|
||||
self.supports = []
|
||||
self.claims = []
|
||||
self.tags = []
|
||||
self.update_claims = []
|
||||
self.delete_tags = []
|
||||
self.tx_filters = []
|
||||
self.block_filters = []
|
||||
|
||||
@staticmethod
|
||||
def block_to_row(block: Block) -> dict:
|
||||
return {
|
||||
'block_hash': block.block_hash,
|
||||
'previous_hash': block.prev_block_hash,
|
||||
'file_number': block.file_number,
|
||||
'height': 0 if block.is_first_block else block.height,
|
||||
'timestamp': block.timestamp,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def tx_to_row(block_hash: bytes, tx: Transaction) -> dict:
|
||||
row = {
|
||||
'tx_hash': tx.hash,
|
||||
#'block_hash': block_hash,
|
||||
'raw': tx.raw,
|
||||
'height': tx.height,
|
||||
'position': tx.position,
|
||||
'is_verified': tx.is_verified,
|
||||
'timestamp': tx.timestamp,
|
||||
'day': tx.day,
|
||||
'purchased_claim_hash': None,
|
||||
}
|
||||
txos = tx.outputs
|
||||
if len(txos) >= 2 and txos[1].can_decode_purchase_data:
|
||||
txos[0].purchase = txos[1]
|
||||
row['purchased_claim_hash'] = txos[1].purchase_data.claim_hash
|
||||
return row
|
||||
|
||||
@staticmethod
|
||||
def txi_to_row(tx: Transaction, txi: Input) -> dict:
|
||||
return {
|
||||
'tx_hash': tx.hash,
|
||||
'txo_hash': txi.txo_ref.hash,
|
||||
'position': txi.position,
|
||||
'height': tx.height,
|
||||
}
|
||||
|
||||
def txo_to_row(self, tx: Transaction, txo: Output) -> dict:
|
||||
row = {
|
||||
'tx_hash': tx.hash,
|
||||
'txo_hash': txo.hash,
|
||||
'address': txo.get_address(self.ledger) if txo.has_address else None,
|
||||
'position': txo.position,
|
||||
'amount': txo.amount,
|
||||
'height': tx.height,
|
||||
'script_offset': txo.script.offset,
|
||||
'script_length': txo.script.length,
|
||||
'txo_type': 0,
|
||||
'claim_id': None,
|
||||
'claim_hash': None,
|
||||
'claim_name': None,
|
||||
'channel_hash': None,
|
||||
'signature': None,
|
||||
'signature_digest': None,
|
||||
'reposted_claim_hash': None,
|
||||
'public_key': None,
|
||||
'public_key_hash': None
|
||||
}
|
||||
if txo.is_claim:
|
||||
if txo.can_decode_claim:
|
||||
claim = txo.claim
|
||||
row['txo_type'] = TXO_TYPES.get(claim.claim_type, TXO_TYPES['stream'])
|
||||
if claim.is_channel:
|
||||
row['public_key'] = claim.channel.public_key_bytes
|
||||
row['public_key_hash'] = self.ledger.address_to_hash160(
|
||||
self.ledger.public_key_to_address(claim.channel.public_key_bytes)
|
||||
)
|
||||
elif claim.is_repost:
|
||||
row['reposted_claim_hash'] = claim.repost.reference.claim_hash
|
||||
else:
|
||||
row['txo_type'] = TXO_TYPES['stream']
|
||||
elif txo.is_support:
|
||||
row['txo_type'] = TXO_TYPES['support']
|
||||
elif txo.purchase is not None:
|
||||
row['txo_type'] = TXO_TYPES['purchase']
|
||||
row['claim_id'] = txo.purchased_claim_id
|
||||
row['claim_hash'] = txo.purchased_claim_hash
|
||||
if txo.script.is_claim_involved:
|
||||
signable = txo.can_decode_signable
|
||||
if signable and signable.is_signed:
|
||||
row['channel_hash'] = signable.signing_channel_hash
|
||||
row['signature'] = txo.get_encoded_signature()
|
||||
row['signature_digest'] = txo.get_signature_digest(self.ledger)
|
||||
row['claim_id'] = txo.claim_id
|
||||
row['claim_hash'] = txo.claim_hash
|
||||
try:
|
||||
row['claim_name'] = txo.claim_name.replace('\x00', '')
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
return row
|
||||
|
||||
def claim_to_rows(
|
||||
self, txo: Output, staked_support_amount: int, staked_support_count: int, reposted_count: int,
|
||||
signature: bytes = None, signature_digest: bytes = None, channel_public_key: bytes = None,
|
||||
) -> Tuple[dict, List]:
|
||||
|
||||
tx = txo.tx_ref
|
||||
d = {
|
||||
'claim_type': None,
|
||||
'address': txo.get_address(self.ledger),
|
||||
'txo_hash': txo.hash,
|
||||
'amount': txo.amount,
|
||||
'height': tx.height,
|
||||
'timestamp': tx.timestamp,
|
||||
# support
|
||||
'staked_amount': txo.amount + staked_support_amount,
|
||||
'staked_support_amount': staked_support_amount,
|
||||
'staked_support_count': staked_support_count,
|
||||
# basic metadata
|
||||
'title': None,
|
||||
'description': None,
|
||||
'author': None,
|
||||
# streams
|
||||
'stream_type': None,
|
||||
'media_type': None,
|
||||
'duration': None,
|
||||
'release_time': None,
|
||||
'fee_amount': 0,
|
||||
'fee_currency': None,
|
||||
# reposts
|
||||
'reposted_claim_hash': None,
|
||||
'reposted_count': reposted_count,
|
||||
# signed claims
|
||||
'channel_hash': None,
|
||||
'is_signature_valid': None,
|
||||
# channels (on last change) and streams (on last re-validation)
|
||||
'public_key_hash': None,
|
||||
'public_key_height': None,
|
||||
}
|
||||
|
||||
claim = txo.can_decode_claim
|
||||
if not claim:
|
||||
return d, []
|
||||
|
||||
if claim.is_stream:
|
||||
d['claim_type'] = TXO_TYPES['stream']
|
||||
d['media_type'] = claim.stream.source.media_type
|
||||
d['stream_type'] = STREAM_TYPES[guess_stream_type(d['media_type'])]
|
||||
d['title'] = claim.stream.title.replace('\x00', '')
|
||||
d['description'] = claim.stream.description.replace('\x00', '')
|
||||
d['author'] = claim.stream.author.replace('\x00', '')
|
||||
if claim.stream.video and claim.stream.video.duration:
|
||||
d['duration'] = claim.stream.video.duration
|
||||
if claim.stream.audio and claim.stream.audio.duration:
|
||||
d['duration'] = claim.stream.audio.duration
|
||||
if claim.stream.release_time:
|
||||
d['release_time'] = claim.stream.release_time
|
||||
if claim.stream.has_fee:
|
||||
fee = claim.stream.fee
|
||||
if isinstance(fee.amount, Decimal):
|
||||
d['fee_amount'] = int(fee.amount*1000)
|
||||
if isinstance(fee.currency, str):
|
||||
d['fee_currency'] = fee.currency.lower()
|
||||
elif claim.is_repost:
|
||||
d['claim_type'] = TXO_TYPES['repost']
|
||||
d['reposted_claim_hash'] = claim.repost.reference.claim_hash
|
||||
elif claim.is_channel:
|
||||
d['claim_type'] = TXO_TYPES['channel']
|
||||
d['public_key_hash'] = self.ledger.address_to_hash160(
|
||||
self.ledger.public_key_to_address(claim.channel.public_key_bytes)
|
||||
)
|
||||
if claim.is_signed:
|
||||
d['channel_hash'] = claim.signing_channel_hash
|
||||
d['is_signature_valid'] = (
|
||||
all((signature, signature_digest, channel_public_key)) and
|
||||
Output.is_signature_valid(
|
||||
signature, signature_digest, channel_public_key
|
||||
)
|
||||
)
|
||||
if channel_public_key:
|
||||
d['public_key_hash'] = self.ledger.address_to_hash160(
|
||||
self.ledger.public_key_to_address(channel_public_key)
|
||||
)
|
||||
|
||||
tags = []
|
||||
if claim.message.tags:
|
||||
claim_hash = txo.claim_hash
|
||||
tags = [
|
||||
{'claim_hash': claim_hash, 'tag': tag}
|
||||
for tag in clean_tags(claim.message.tags)
|
||||
]
|
||||
|
||||
return d, tags
|
||||
|
||||
def support_to_row(
|
||||
self, txo: Output, channel_public_key: bytes = None,
|
||||
signature: bytes = None, signature_digest: bytes = None
|
||||
):
|
||||
tx = txo.tx_ref
|
||||
d = {
|
||||
'txo_hash': txo.ref.hash,
|
||||
'claim_hash': txo.claim_hash,
|
||||
'address': txo.get_address(self.ledger),
|
||||
'amount': txo.amount,
|
||||
'height': tx.height,
|
||||
'timestamp': tx.timestamp,
|
||||
'emoji': None,
|
||||
'channel_hash': None,
|
||||
'is_signature_valid': None,
|
||||
}
|
||||
support = txo.can_decode_support
|
||||
if support:
|
||||
d['emoji'] = support.emoji
|
||||
if support.is_signed:
|
||||
d['channel_hash'] = support.signing_channel_hash
|
||||
d['is_signature_valid'] = (
|
||||
all((signature, signature_digest, channel_public_key)) and
|
||||
Output.is_signature_valid(
|
||||
signature, signature_digest, channel_public_key
|
||||
)
|
||||
)
|
||||
return d
|
||||
|
||||
def add_block(self, block: Block):
|
||||
self.blocks.append(self.block_to_row(block))
|
||||
for tx in block.txs:
|
||||
self.add_transaction(block.block_hash, tx)
|
||||
return self
|
||||
|
||||
def add_block_filter(self, height: int, factor: int, address_filter: bytes):
|
||||
self.block_filters.append({
|
||||
'height': height,
|
||||
'factor': factor,
|
||||
'address_filter': address_filter
|
||||
})
|
||||
|
||||
def add_transaction(self, block_hash: bytes, tx: Transaction):
|
||||
self.txs.append(self.tx_to_row(block_hash, tx))
|
||||
for txi in tx.inputs:
|
||||
if txi.coinbase is None:
|
||||
self.txis.append(self.txi_to_row(tx, txi))
|
||||
for txo in tx.outputs:
|
||||
self.txos.append(self.txo_to_row(tx, txo))
|
||||
return self
|
||||
|
||||
def add_transaction_filter(self, tx_hash: bytes, height: int, address_filter: bytes):
|
||||
self.tx_filters.append({
|
||||
'tx_hash': tx_hash,
|
||||
'height': height,
|
||||
'address_filter': address_filter
|
||||
})
|
||||
|
||||
def add_support(self, txo: Output, **extra):
|
||||
self.supports.append(self.support_to_row(txo, **extra))
|
||||
|
||||
def add_claim(
|
||||
self, txo: Output, short_url: str,
|
||||
creation_height: int, activation_height: int, expiration_height: int,
|
||||
takeover_height: int = None, **extra
|
||||
):
|
||||
try:
|
||||
claim_name = txo.claim_name.replace('\x00', '')
|
||||
normalized_name = txo.normalized_name
|
||||
except UnicodeDecodeError:
|
||||
claim_name = normalized_name = ''
|
||||
d, tags = self.claim_to_rows(txo, **extra)
|
||||
d['claim_hash'] = txo.claim_hash
|
||||
d['claim_id'] = txo.claim_id
|
||||
d['claim_name'] = claim_name
|
||||
d['normalized'] = normalized_name
|
||||
d['short_url'] = short_url
|
||||
d['creation_height'] = creation_height
|
||||
d['activation_height'] = activation_height
|
||||
d['expiration_height'] = expiration_height
|
||||
d['takeover_height'] = takeover_height
|
||||
d['is_controlling'] = takeover_height is not None
|
||||
if d['public_key_hash'] is not None:
|
||||
d['public_key_height'] = d['height']
|
||||
self.claims.append(d)
|
||||
self.tags.extend(tags)
|
||||
return self
|
||||
|
||||
def update_claim(self, txo: Output, public_key_height=None, **extra):
|
||||
d, tags = self.claim_to_rows(txo, **extra)
|
||||
d['pk'] = txo.claim_hash
|
||||
d['_public_key_height'] = public_key_height or d['height']
|
||||
d['_public_key_hash'] = d['public_key_hash']
|
||||
self.update_claims.append(d)
|
||||
self.delete_tags.append({'pk': txo.claim_hash})
|
||||
self.tags.extend(tags)
|
||||
return self
|
||||
|
||||
def get_queries(self):
|
||||
return (
|
||||
(Block.insert(), self.blocks),
|
||||
(BlockFilter.insert(), self.block_filters),
|
||||
(TX.insert(), self.txs),
|
||||
(TXFilter.insert(), self.tx_filters),
|
||||
(TXO.insert(), self.txos),
|
||||
(TXI.insert(), self.txis),
|
||||
(Claim.insert(), self.claims),
|
||||
(Tag.delete().where(Tag.c.claim_hash == bindparam('pk')), self.delete_tags),
|
||||
(Claim.update()
|
||||
.values(public_key_height=case([
|
||||
(bindparam('_public_key_hash').is_(None), None),
|
||||
(Claim.c.public_key_hash.is_(None) |
|
||||
(Claim.c.public_key_hash != bindparam('_public_key_hash')),
|
||||
bindparam('_public_key_height')),
|
||||
], else_=Claim.c.public_key_height))
|
||||
.where(Claim.c.claim_hash == bindparam('pk')),
|
||||
self.update_claims),
|
||||
(Tag.insert(), self.tags),
|
||||
(Support.insert(), self.supports),
|
||||
)
|
||||
|
||||
def flush(self, return_row_count_for_table) -> int:
|
||||
done = 0
|
||||
for sql, rows in self.get_queries():
|
||||
if not rows:
|
||||
continue
|
||||
if self.ctx.is_postgres and isinstance(sql, Insert):
|
||||
self.ctx.pg_copy(sql.table, rows)
|
||||
else:
|
||||
self.ctx.execute(sql, rows)
|
||||
if sql.table == return_row_count_for_table:
|
||||
done += len(rows)
|
||||
rows.clear()
|
||||
return done
|
||||
-103
@@ -1,103 +0,0 @@
|
||||
from sqlalchemy.future import select
|
||||
|
||||
from lbry.db.query_context import progress, Event
|
||||
from lbry.db.tables import TX, TXI, TXO, Claim, Support
|
||||
from .constants import TXO_TYPES, CLAIM_TYPE_CODES
|
||||
from .queries import (
|
||||
BASE_SELECT_TXO_COLUMNS,
|
||||
rows_to_txos, where_unspent_txos,
|
||||
where_abandoned_supports,
|
||||
where_abandoned_claims
|
||||
)
|
||||
|
||||
|
||||
SPENDS_UPDATE_EVENT = Event.add("client.sync.spends.update", "steps")
|
||||
CLAIMS_INSERT_EVENT = Event.add("client.sync.claims.insert", "claims")
|
||||
CLAIMS_UPDATE_EVENT = Event.add("client.sync.claims.update", "claims")
|
||||
CLAIMS_DELETE_EVENT = Event.add("client.sync.claims.delete", "claims")
|
||||
SUPPORT_INSERT_EVENT = Event.add("client.sync.supports.insert", "supports")
|
||||
SUPPORT_UPDATE_EVENT = Event.add("client.sync.supports.update", "supports")
|
||||
SUPPORT_DELETE_EVENT = Event.add("client.sync.supports.delete", "supports")
|
||||
|
||||
|
||||
def process_all_things_after_sync():
|
||||
with progress(SPENDS_UPDATE_EVENT) as p:
|
||||
p.start(2)
|
||||
update_spent_outputs(p.ctx)
|
||||
p.step(1)
|
||||
set_input_addresses(p.ctx)
|
||||
p.step(2)
|
||||
with progress(SUPPORT_DELETE_EVENT) as p:
|
||||
p.start(1)
|
||||
sql = Support.delete().where(where_abandoned_supports())
|
||||
p.ctx.execute(sql)
|
||||
with progress(SUPPORT_INSERT_EVENT) as p:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
sql = (
|
||||
select(*BASE_SELECT_TXO_COLUMNS)
|
||||
.where(where_unspent_txos(TXO_TYPES['support'], missing_in_supports_table=True))
|
||||
.select_from(TXO.join(TX))
|
||||
)
|
||||
for support in rows_to_txos(p.ctx.fetchall(sql)):
|
||||
loader.add_support(support)
|
||||
loader.flush(Support)
|
||||
with progress(CLAIMS_DELETE_EVENT) as p:
|
||||
p.start(1)
|
||||
sql = Claim.delete().where(where_abandoned_claims())
|
||||
p.ctx.execute(sql)
|
||||
with progress(CLAIMS_INSERT_EVENT) as p:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
sql = (
|
||||
select(*BASE_SELECT_TXO_COLUMNS)
|
||||
.where(where_unspent_txos(CLAIM_TYPE_CODES, missing_in_claims_table=True))
|
||||
.select_from(TXO.join(TX))
|
||||
)
|
||||
for claim in rows_to_txos(p.ctx.fetchall(sql)):
|
||||
loader.add_claim(claim, '', 0, 0, 0, 0, staked_support_amount=0, staked_support_count=0)
|
||||
loader.flush(Claim)
|
||||
with progress(CLAIMS_UPDATE_EVENT) as p:
|
||||
loader = p.ctx.get_bulk_loader()
|
||||
sql = (
|
||||
select(*BASE_SELECT_TXO_COLUMNS)
|
||||
.where(where_unspent_txos(CLAIM_TYPE_CODES, missing_or_stale_in_claims_table=True))
|
||||
.select_from(TXO.join(TX))
|
||||
)
|
||||
for claim in rows_to_txos(p.ctx.fetchall(sql)):
|
||||
loader.update_claim(claim)
|
||||
loader.flush(Claim)
|
||||
|
||||
|
||||
def set_input_addresses(ctx):
|
||||
# Update TXIs to have the address of TXO they are spending.
|
||||
if ctx.is_sqlite:
|
||||
address_query = select(TXO.c.address).where(TXI.c.txo_hash == TXO.c.txo_hash)
|
||||
set_addresses = (
|
||||
TXI.update()
|
||||
.values(address=address_query.scalar_subquery())
|
||||
.where(TXI.c.address.is_(None))
|
||||
)
|
||||
else:
|
||||
set_addresses = (
|
||||
TXI.update()
|
||||
.values({TXI.c.address: TXO.c.address})
|
||||
.where((TXI.c.address.is_(None)) & (TXI.c.txo_hash == TXO.c.txo_hash))
|
||||
)
|
||||
ctx.execute(set_addresses)
|
||||
|
||||
|
||||
def update_spent_outputs(ctx):
|
||||
# Update spent TXOs setting spent_height
|
||||
set_spent_height = (
|
||||
TXO.update()
|
||||
.values({
|
||||
TXO.c.spent_height: (
|
||||
select(TXI.c.height)
|
||||
.where(TXI.c.txo_hash == TXO.c.txo_hash)
|
||||
.scalar_subquery()
|
||||
)
|
||||
}).where(
|
||||
(TXO.c.spent_height == 0) &
|
||||
(TXO.c.txo_hash.in_(select(TXI.c.txo_hash).where(TXI.c.address.is_(None))))
|
||||
)
|
||||
)
|
||||
ctx.execute(set_spent_height)
|
||||
@@ -1,331 +0,0 @@
|
||||
# pylint: skip-file
|
||||
|
||||
from sqlalchemy import (
|
||||
MetaData, Table, Column, ForeignKey,
|
||||
LargeBinary, Text, SmallInteger, Integer, BigInteger, Boolean,
|
||||
)
|
||||
from .constants import TXO_TYPES, CLAIM_TYPE_CODES
|
||||
|
||||
|
||||
SCHEMA_VERSION = '1.4'
|
||||
|
||||
|
||||
metadata = MetaData()
|
||||
|
||||
|
||||
Version = Table(
|
||||
'version', metadata,
|
||||
Column('version', Text, primary_key=True),
|
||||
)
|
||||
|
||||
|
||||
Wallet = Table(
|
||||
'wallet', metadata,
|
||||
Column('wallet_id', Text, primary_key=True),
|
||||
Column('data', Text),
|
||||
)
|
||||
|
||||
|
||||
PubkeyAddress = Table(
|
||||
'pubkey_address', metadata,
|
||||
Column('address', Text, primary_key=True),
|
||||
Column('used_times', Integer, server_default='0'),
|
||||
)
|
||||
|
||||
|
||||
AccountAddress = Table(
|
||||
'account_address', metadata,
|
||||
Column('account', Text, primary_key=True),
|
||||
Column('address', Text, ForeignKey(PubkeyAddress.columns.address), primary_key=True),
|
||||
Column('chain', SmallInteger),
|
||||
Column('pubkey', LargeBinary),
|
||||
Column('chain_code', LargeBinary),
|
||||
Column('n', Integer),
|
||||
Column('depth', SmallInteger),
|
||||
)
|
||||
|
||||
|
||||
pg_add_account_address_constraints_and_indexes = [
|
||||
"CREATE UNIQUE INDEX account_address_idx ON account_address (account, address);"
|
||||
]
|
||||
|
||||
|
||||
Block = Table(
|
||||
'block', metadata,
|
||||
Column('height', Integer, primary_key=True),
|
||||
Column('block_hash', LargeBinary),
|
||||
Column('previous_hash', LargeBinary),
|
||||
Column('file_number', SmallInteger),
|
||||
Column('timestamp', Integer),
|
||||
)
|
||||
|
||||
pg_add_block_constraints_and_indexes = [
|
||||
"ALTER TABLE block ADD PRIMARY KEY (height);",
|
||||
]
|
||||
|
||||
|
||||
BlockFilter = Table(
|
||||
'block_filter', metadata,
|
||||
Column('height', Integer),
|
||||
Column('factor', SmallInteger),
|
||||
Column('address_filter', LargeBinary),
|
||||
)
|
||||
|
||||
pg_add_block_filter_constraints_and_indexes = [
|
||||
"ALTER TABLE block_filter ADD PRIMARY KEY (height, factor);",
|
||||
]
|
||||
|
||||
|
||||
TX = Table(
|
||||
'tx', metadata,
|
||||
Column('tx_hash', LargeBinary, primary_key=True),
|
||||
Column('raw', LargeBinary),
|
||||
Column('height', Integer),
|
||||
Column('position', SmallInteger),
|
||||
Column('timestamp', Integer, nullable=True),
|
||||
Column('day', Integer, nullable=True),
|
||||
Column('is_verified', Boolean, server_default='FALSE'),
|
||||
Column('purchased_claim_hash', LargeBinary, nullable=True),
|
||||
)
|
||||
|
||||
pg_add_tx_constraints_and_indexes = [
|
||||
"ALTER TABLE tx ADD PRIMARY KEY (tx_hash);",
|
||||
]
|
||||
|
||||
|
||||
TXFilter = Table(
|
||||
'tx_filter', metadata,
|
||||
Column('tx_hash', LargeBinary, primary_key=True),
|
||||
Column('height', Integer),
|
||||
Column('address_filter', LargeBinary),
|
||||
)
|
||||
|
||||
pg_add_tx_filter_constraints_and_indexes = [
|
||||
"ALTER TABLE tx_filter ADD PRIMARY KEY (tx_hash);",
|
||||
"ALTER TABLE tx_filter ADD CONSTRAINT fk_tx_filter"
|
||||
" FOREIGN KEY (tx_hash) REFERENCES tx (tx_hash) ON DELETE CASCADE;"
|
||||
]
|
||||
|
||||
|
||||
TXO = Table(
|
||||
'txo', metadata,
|
||||
Column('tx_hash', LargeBinary, ForeignKey(TX.columns.tx_hash)),
|
||||
Column('txo_hash', LargeBinary, primary_key=True),
|
||||
Column('address', Text),
|
||||
Column('position', SmallInteger),
|
||||
Column('amount', BigInteger),
|
||||
Column('height', Integer),
|
||||
Column('spent_height', Integer, server_default='0'),
|
||||
Column('script_offset', Integer),
|
||||
Column('script_length', Integer),
|
||||
Column('is_reserved', Boolean, server_default='0'),
|
||||
|
||||
# claims
|
||||
Column('txo_type', SmallInteger, server_default='0'),
|
||||
Column('claim_id', Text, nullable=True),
|
||||
Column('claim_hash', LargeBinary, nullable=True),
|
||||
Column('claim_name', Text, nullable=True),
|
||||
Column('channel_hash', LargeBinary, nullable=True), # claims in channel
|
||||
Column('signature', LargeBinary, nullable=True),
|
||||
Column('signature_digest', LargeBinary, nullable=True),
|
||||
|
||||
# reposts
|
||||
Column('reposted_claim_hash', LargeBinary, nullable=True),
|
||||
|
||||
# channels
|
||||
Column('public_key', LargeBinary, nullable=True),
|
||||
Column('public_key_hash', LargeBinary, nullable=True),
|
||||
)
|
||||
|
||||
txo_join_account = TXO.join(AccountAddress, TXO.columns.address == AccountAddress.columns.address)
|
||||
|
||||
pg_add_txo_constraints_and_indexes = [
|
||||
"ALTER TABLE txo ADD PRIMARY KEY (txo_hash);",
|
||||
# find appropriate channel public key for signing a content claim
|
||||
f"CREATE INDEX txo_channel_hash_by_height_desc_w_pub_key "
|
||||
f"ON txo (claim_hash, height desc) INCLUDE (public_key) "
|
||||
f"WHERE txo_type={TXO_TYPES['channel']};",
|
||||
# for calculating supports on a claim
|
||||
f"CREATE INDEX txo_unspent_supports ON txo (claim_hash) INCLUDE (amount) "
|
||||
f"WHERE spent_height = 0 AND txo_type={TXO_TYPES['support']};",
|
||||
# for calculating balance
|
||||
f"CREATE INDEX txo_unspent_by_address ON txo (address) INCLUDE (amount, txo_type, tx_hash) "
|
||||
f"WHERE spent_height = 0;",
|
||||
# for finding modified claims in a block range
|
||||
f"CREATE INDEX txo_claim_changes "
|
||||
f"ON txo (height DESC) INCLUDE (claim_hash, txo_hash) "
|
||||
f"WHERE spent_height = 0 AND txo_type IN {tuple(CLAIM_TYPE_CODES)};",
|
||||
# for finding claims which need support totals re-calculated in a block range
|
||||
f"CREATE INDEX txo_added_supports_by_height ON txo (height DESC) "
|
||||
f"INCLUDE (claim_hash) WHERE txo_type={TXO_TYPES['support']};",
|
||||
f"CREATE INDEX txo_spent_supports_by_height ON txo (spent_height DESC) "
|
||||
f"INCLUDE (claim_hash) WHERE txo_type={TXO_TYPES['support']};",
|
||||
# for finding claims which need repost totals re-calculated in a block range
|
||||
f"CREATE INDEX txo_added_reposts_by_height ON txo (height DESC) "
|
||||
f"INCLUDE (reposted_claim_hash) WHERE txo_type={TXO_TYPES['repost']};",
|
||||
f"CREATE INDEX txo_spent_reposts_by_height ON txo (spent_height DESC) "
|
||||
f"INCLUDE (reposted_claim_hash) WHERE txo_type={TXO_TYPES['repost']};",
|
||||
"CREATE INDEX txo_reposted_claim_hash ON txo (reposted_claim_hash)"
|
||||
"WHERE reposted_claim_hash IS NOT NULL AND spent_height = 0;",
|
||||
"CREATE INDEX txo_height ON txo (height);",
|
||||
# used by sum_supports query (at least)
|
||||
"CREATE INDEX txo_claim_hash ON txo (claim_hash)",
|
||||
]
|
||||
|
||||
|
||||
TXI = Table(
|
||||
'txi', metadata,
|
||||
Column('tx_hash', LargeBinary, ForeignKey(TX.columns.tx_hash)),
|
||||
Column('txo_hash', LargeBinary, ForeignKey(TXO.columns.txo_hash), primary_key=True),
|
||||
Column('address', Text, nullable=True),
|
||||
Column('position', SmallInteger),
|
||||
Column('height', Integer),
|
||||
)
|
||||
|
||||
txi_join_account = TXI.join(AccountAddress, TXI.columns.address == AccountAddress.columns.address)
|
||||
|
||||
pg_add_txi_constraints_and_indexes = [
|
||||
"ALTER TABLE txi ADD PRIMARY KEY (txo_hash);",
|
||||
"CREATE INDEX txi_height ON txi (height);",
|
||||
"CREATE INDEX txi_first_address ON txi (tx_hash) INCLUDE (address) WHERE position = 0;",
|
||||
]
|
||||
|
||||
|
||||
Claim = Table(
|
||||
'claim', metadata,
|
||||
Column('claim_hash', LargeBinary, primary_key=True),
|
||||
Column('claim_id', Text),
|
||||
Column('claim_name', Text),
|
||||
Column('normalized', Text),
|
||||
Column('address', Text),
|
||||
Column('txo_hash', LargeBinary, ForeignKey(TXO.columns.txo_hash)),
|
||||
Column('amount', BigInteger),
|
||||
Column('staked_amount', BigInteger),
|
||||
Column('timestamp', Integer), # last updated timestamp
|
||||
Column('creation_timestamp', Integer),
|
||||
Column('release_time', BigInteger, nullable=True),
|
||||
Column('height', Integer), # last updated height
|
||||
Column('creation_height', Integer),
|
||||
Column('activation_height', Integer),
|
||||
Column('expiration_height', Integer),
|
||||
Column('takeover_height', Integer, nullable=True),
|
||||
Column('is_controlling', Boolean),
|
||||
|
||||
# short_url: normalized#shortest-unique-claim_id
|
||||
Column('short_url', Text),
|
||||
# canonical_url: channel's-short_url/normalized#shortest-unique-claim_id-within-channel
|
||||
# canonical_url is computed dynamically
|
||||
|
||||
Column('title', Text, nullable=True),
|
||||
Column('author', Text, nullable=True),
|
||||
Column('description', Text, nullable=True),
|
||||
|
||||
Column('claim_type', SmallInteger),
|
||||
Column('staked_support_count', Integer, server_default='0'),
|
||||
Column('staked_support_amount', BigInteger, server_default='0'),
|
||||
|
||||
# streams
|
||||
Column('stream_type', SmallInteger, nullable=True),
|
||||
Column('media_type', Text, nullable=True),
|
||||
Column('fee_amount', BigInteger, server_default='0'),
|
||||
Column('fee_currency', Text, nullable=True),
|
||||
Column('duration', Integer, nullable=True),
|
||||
|
||||
# reposts
|
||||
Column('reposted_claim_hash', LargeBinary, nullable=True), # on claim doing the repost
|
||||
Column('reposted_count', Integer, server_default='0'), # on claim being reposted
|
||||
|
||||
# claims which are channels
|
||||
Column('signed_claim_count', Integer, server_default='0'),
|
||||
Column('signed_support_count', Integer, server_default='0'),
|
||||
Column('public_key_hash', LargeBinary, nullable=True), # included for claims in channel as well
|
||||
Column('public_key_height', Integer, nullable=True), # last updated height
|
||||
|
||||
# claims which are inside channels
|
||||
Column('channel_hash', LargeBinary, nullable=True),
|
||||
Column('is_signature_valid', Boolean, nullable=True),
|
||||
)
|
||||
|
||||
Tag = Table(
|
||||
'tag', metadata,
|
||||
Column('claim_hash', LargeBinary),
|
||||
Column('tag', Text),
|
||||
)
|
||||
|
||||
pg_add_claim_and_tag_constraints_and_indexes = [
|
||||
"ALTER TABLE claim ADD PRIMARY KEY (claim_hash);",
|
||||
# for checking if claim is up-to-date
|
||||
"CREATE UNIQUE INDEX claim_txo_hash ON claim (txo_hash);",
|
||||
# used by takeover process to reset winning claims
|
||||
"CREATE INDEX claim_normalized ON claim (normalized);",
|
||||
# ordering and search by release_time
|
||||
"CREATE INDEX claim_release_time ON claim (release_time DESC NULLs LAST);",
|
||||
# used to count()/sum() claims signed by channel
|
||||
"CREATE INDEX signed_content ON claim (channel_hash) "
|
||||
"INCLUDE (amount) WHERE is_signature_valid;",
|
||||
# used to count()/sum() reposted claims
|
||||
"CREATE INDEX reposted_content ON claim (reposted_claim_hash);",
|
||||
# basic tag indexes
|
||||
"ALTER TABLE tag ADD PRIMARY KEY (claim_hash, tag);",
|
||||
"CREATE INDEX tags ON tag (tag) INCLUDE (claim_hash);",
|
||||
# used by sum_supports query (at least)
|
||||
"CREATE INDEX claim_channel_hash ON claim (channel_hash)",
|
||||
]
|
||||
|
||||
|
||||
Support = Table(
|
||||
'support', metadata,
|
||||
|
||||
Column('txo_hash', LargeBinary, ForeignKey(TXO.columns.txo_hash), primary_key=True),
|
||||
Column('claim_hash', LargeBinary),
|
||||
Column('address', Text),
|
||||
Column('amount', BigInteger),
|
||||
Column('height', Integer),
|
||||
Column('timestamp', Integer),
|
||||
|
||||
# support metadata
|
||||
Column('emoji', Text),
|
||||
|
||||
# signed supports
|
||||
Column('channel_hash', LargeBinary, nullable=True),
|
||||
Column('signature', LargeBinary, nullable=True),
|
||||
Column('signature_digest', LargeBinary, nullable=True),
|
||||
Column('is_signature_valid', Boolean, nullable=True),
|
||||
)
|
||||
|
||||
pg_add_support_constraints_and_indexes = [
|
||||
"ALTER TABLE support ADD PRIMARY KEY (txo_hash);",
|
||||
# used to count()/sum() supports signed by channel
|
||||
"CREATE INDEX signed_support ON support (channel_hash) "
|
||||
"INCLUDE (amount) WHERE is_signature_valid;",
|
||||
]
|
||||
|
||||
|
||||
Stake = Table(
|
||||
'stake', metadata,
|
||||
Column('claim_hash', LargeBinary),
|
||||
Column('height', Integer),
|
||||
Column('stake_min', BigInteger),
|
||||
Column('stake_max', BigInteger),
|
||||
Column('stake_sum', BigInteger),
|
||||
Column('stake_count', Integer),
|
||||
Column('stake_unique', Integer),
|
||||
)
|
||||
|
||||
|
||||
Trend = Table(
|
||||
'trend', metadata,
|
||||
Column('claim_hash', LargeBinary, primary_key=True),
|
||||
Column('trend_group', BigInteger, server_default='0'),
|
||||
Column('trend_mixed', BigInteger, server_default='0'),
|
||||
Column('trend_local', BigInteger, server_default='0'),
|
||||
Column('trend_global', BigInteger, server_default='0'),
|
||||
)
|
||||
|
||||
|
||||
CensoredClaim = Table(
|
||||
'censored_claim', metadata,
|
||||
Column('claim_hash', LargeBinary, primary_key=True),
|
||||
Column('censor_type', SmallInteger),
|
||||
Column('censoring_channel_hash', LargeBinary),
|
||||
)
|
||||
@@ -1,25 +0,0 @@
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from lbry.db.query_context import event_emitter, ProgressContext
|
||||
from lbry.db.tables import Trend, Support, Claim
|
||||
WINDOW = 576 # a day
|
||||
|
||||
|
||||
@event_emitter("blockchain.sync.trending.update", "steps")
|
||||
def calculate_trending(height, p: ProgressContext):
|
||||
with p.ctx.engine.begin() as ctx:
|
||||
ctx.execute(Trend.delete())
|
||||
start = height - WINDOW
|
||||
trending = func.sum(Support.c.amount * (WINDOW - (height - Support.c.height)))
|
||||
sql = (
|
||||
select([Claim.c.claim_hash, trending, trending, trending, 4])
|
||||
.where(
|
||||
(Support.c.claim_hash == Claim.c.claim_hash) &
|
||||
(Support.c.height <= height) &
|
||||
(Support.c.height >= start)
|
||||
).group_by(Claim.c.claim_hash)
|
||||
)
|
||||
ctx.execute(Trend.insert().from_select(
|
||||
['claim_hash', 'trend_global', 'trend_local', 'trend_mixed', 'trend_group'], sql
|
||||
))
|
||||
@@ -1,174 +0,0 @@
|
||||
from itertools import islice
|
||||
from typing import List, Union
|
||||
|
||||
from sqlalchemy import text, and_, or_
|
||||
from sqlalchemy.sql.expression import Select, FunctionElement
|
||||
from sqlalchemy.types import Numeric
|
||||
from sqlalchemy.ext.compiler import compiles
|
||||
try:
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert # pylint: disable=unused-import
|
||||
except ImportError:
|
||||
pg_insert = None
|
||||
|
||||
from .tables import AccountAddress
|
||||
|
||||
|
||||
class greatest(FunctionElement): # pylint: disable=invalid-name
|
||||
type = Numeric()
|
||||
name = 'greatest'
|
||||
|
||||
|
||||
@compiles(greatest)
|
||||
def default_greatest(element, compiler, **kw):
|
||||
return "greatest(%s)" % compiler.process(element.clauses, **kw)
|
||||
|
||||
|
||||
@compiles(greatest, 'sqlite')
|
||||
def sqlite_greatest(element, compiler, **kw):
|
||||
return "max(%s)" % compiler.process(element.clauses, **kw)
|
||||
|
||||
|
||||
class least(FunctionElement): # pylint: disable=invalid-name
|
||||
type = Numeric()
|
||||
name = 'least'
|
||||
|
||||
|
||||
@compiles(least)
|
||||
def default_least(element, compiler, **kw):
|
||||
return "least(%s)" % compiler.process(element.clauses, **kw)
|
||||
|
||||
|
||||
@compiles(least, 'sqlite')
|
||||
def sqlite_least(element, compiler, **kw):
|
||||
return "min(%s)" % compiler.process(element.clauses, **kw)
|
||||
|
||||
|
||||
def chunk(rows, step):
|
||||
it, total = iter(rows), len(rows)
|
||||
for _ in range(0, total, step):
|
||||
yield list(islice(it, step))
|
||||
total -= step
|
||||
|
||||
|
||||
def constrain_single_or_list(constraints, column, value, convert=lambda x: x):
|
||||
if value is not None:
|
||||
if isinstance(value, list):
|
||||
value = [convert(v) for v in value]
|
||||
if len(value) == 1:
|
||||
constraints[column] = value[0]
|
||||
elif len(value) > 1:
|
||||
constraints[f"{column}__in"] = value
|
||||
else:
|
||||
constraints[column] = convert(value)
|
||||
return constraints
|
||||
|
||||
|
||||
def in_account_ids(account_ids: Union[List[str], str]):
|
||||
if isinstance(account_ids, list):
|
||||
if len(account_ids) > 1:
|
||||
return AccountAddress.c.account.in_(account_ids)
|
||||
account_ids = account_ids[0]
|
||||
return AccountAddress.c.account == account_ids
|
||||
|
||||
|
||||
def query(table, s: Select, **constraints) -> Select:
|
||||
limit = constraints.pop('limit', None)
|
||||
if limit is not None:
|
||||
s = s.limit(limit)
|
||||
|
||||
offset = constraints.pop('offset', None)
|
||||
if offset is not None:
|
||||
s = s.offset(offset)
|
||||
|
||||
order_by = constraints.pop('order_by', None)
|
||||
if order_by:
|
||||
if isinstance(order_by, str):
|
||||
s = s.order_by(text(order_by))
|
||||
elif isinstance(order_by, list):
|
||||
s = s.order_by(text(', '.join(order_by)))
|
||||
else:
|
||||
raise ValueError("order_by must be string or list")
|
||||
|
||||
group_by = constraints.pop('group_by', None)
|
||||
if group_by is not None:
|
||||
s = s.group_by(text(group_by))
|
||||
|
||||
account_ids = constraints.pop('account_ids', [])
|
||||
if account_ids:
|
||||
s = s.where(in_account_ids(account_ids))
|
||||
|
||||
if constraints:
|
||||
s = s.where(and_(*constraints_to_clause(table, constraints)))
|
||||
|
||||
return s
|
||||
|
||||
|
||||
def constraints_to_clause(tables, constraints):
|
||||
clause = []
|
||||
for key, constraint in constraints.items():
|
||||
if key.endswith('__not'):
|
||||
col, op = key[:-len('__not')], '__ne__'
|
||||
elif key.endswith('__is_null'):
|
||||
col = key[:-len('__is_null')]
|
||||
op = '__eq__'
|
||||
constraint = None
|
||||
elif key.endswith('__is_not_null'):
|
||||
col = key[:-len('__is_not_null')]
|
||||
op = '__ne__'
|
||||
constraint = None
|
||||
elif key.endswith('__lt'):
|
||||
col, op = key[:-len('__lt')], '__lt__'
|
||||
elif key.endswith('__lte'):
|
||||
col, op = key[:-len('__lte')], '__le__'
|
||||
elif key.endswith('__gt'):
|
||||
col, op = key[:-len('__gt')], '__gt__'
|
||||
elif key.endswith('__gte'):
|
||||
col, op = key[:-len('__gte')], '__ge__'
|
||||
elif key.endswith('__like'):
|
||||
col, op = key[:-len('__like')], 'like'
|
||||
elif key.endswith('__not_like'):
|
||||
col, op = key[:-len('__not_like')], 'notlike'
|
||||
elif key.endswith('__in') or key.endswith('__not_in'):
|
||||
if key.endswith('__in'):
|
||||
col, op, one_val_op = key[:-len('__in')], 'in_', '__eq__'
|
||||
else:
|
||||
col, op, one_val_op = key[:-len('__not_in')], 'notin_', '__ne__'
|
||||
if isinstance(constraint, Select):
|
||||
pass
|
||||
elif constraint:
|
||||
if isinstance(constraint, (list, set, tuple)):
|
||||
if len(constraint) == 1:
|
||||
op = one_val_op
|
||||
constraint = next(iter(constraint))
|
||||
elif isinstance(constraint, str):
|
||||
constraint = text(constraint)
|
||||
else:
|
||||
raise ValueError(f"{col} requires a list, set or string as constraint value.")
|
||||
else:
|
||||
continue
|
||||
elif key.endswith('__or'):
|
||||
clause.append(or_(*constraints_to_clause(tables, constraint)))
|
||||
continue
|
||||
else:
|
||||
col, op = key, '__eq__'
|
||||
attr = None
|
||||
if '.' in col:
|
||||
table_name, col = col.split('.')
|
||||
_table = None
|
||||
for table in tables:
|
||||
if table.name == table_name.lower():
|
||||
_table = table
|
||||
break
|
||||
if _table is not None:
|
||||
attr = getattr(_table.c, col)
|
||||
else:
|
||||
raise ValueError(f"Table '{table_name}' not available: {', '.join([t.name for t in tables])}.")
|
||||
else:
|
||||
for table in tables:
|
||||
attr = getattr(table.c, col, None)
|
||||
if attr is not None:
|
||||
break
|
||||
if attr is None:
|
||||
raise ValueError(f"Attribute '{col}' not found on tables: {', '.join([t.name for t in tables])}.")
|
||||
clause.append(getattr(attr, op)(constraint))
|
||||
return clause
|
||||
@@ -1,62 +0,0 @@
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.node import Node
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SQLiteStorage:
|
||||
pass
|
||||
|
||||
|
||||
class BlobAnnouncer:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, node: 'Node', storage: 'SQLiteStorage'):
|
||||
self.loop = loop
|
||||
self.node = node
|
||||
self.storage = storage
|
||||
self.announce_task: asyncio.Task = None
|
||||
self.announce_queue: typing.List[str] = []
|
||||
|
||||
async def _submit_announcement(self, blob_hash):
|
||||
try:
|
||||
peers = len(await self.node.announce_blob(blob_hash))
|
||||
if peers > 4:
|
||||
return blob_hash
|
||||
else:
|
||||
log.debug("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise err
|
||||
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
|
||||
|
||||
async def _announce(self, batch_size: typing.Optional[int] = 10):
|
||||
while batch_size:
|
||||
if not self.node.joined.is_set():
|
||||
await self.node.joined.wait()
|
||||
await asyncio.sleep(60, loop=self.loop)
|
||||
if not self.node.protocol.routing_table.get_peers():
|
||||
log.warning("No peers in DHT, announce round skipped")
|
||||
continue
|
||||
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
|
||||
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
|
||||
while len(self.announce_queue) > 0:
|
||||
log.info("%i blobs to announce", len(self.announce_queue))
|
||||
announced = await asyncio.gather(*[
|
||||
self._submit_announcement(
|
||||
self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
|
||||
], loop=self.loop)
|
||||
announced = list(filter(None, announced))
|
||||
if announced:
|
||||
await self.storage.update_last_announced_blobs(announced)
|
||||
log.info("announced %i blobs", len(announced))
|
||||
|
||||
def start(self, batch_size: typing.Optional[int] = 10):
|
||||
assert not self.announce_task or self.announce_task.done(), "already running"
|
||||
self.announce_task = self.loop.create_task(self._announce(batch_size))
|
||||
|
||||
def stop(self):
|
||||
if self.announce_task and not self.announce_task.done():
|
||||
self.announce_task.cancel()
|
||||
@@ -1,41 +0,0 @@
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
HASH_CLASS = hashlib.sha384 # pylint: disable=invalid-name
|
||||
HASH_LENGTH = HASH_CLASS().digest_size
|
||||
HASH_BITS = HASH_LENGTH * 8
|
||||
ALPHA = 5
|
||||
K = 8
|
||||
SPLIT_BUCKETS_UNDER_INDEX = 1
|
||||
REPLACEMENT_CACHE_SIZE = 8
|
||||
RPC_TIMEOUT = 5.0
|
||||
RPC_ATTEMPTS = 5
|
||||
RPC_ATTEMPTS_PRUNING_WINDOW = 600
|
||||
ITERATIVE_LOOKUP_DELAY = RPC_TIMEOUT / 2.0 # TODO: use config val / 2 if rpc timeout is provided
|
||||
REFRESH_INTERVAL = 3600 # 1 hour
|
||||
REPLICATE_INTERVAL = REFRESH_INTERVAL
|
||||
DATA_EXPIRATION = 86400 # 24 hours
|
||||
TOKEN_SECRET_REFRESH_INTERVAL = 300 # 5 minutes
|
||||
MAYBE_PING_DELAY = 300 # 5 minutes
|
||||
CHECK_REFRESH_INTERVAL = REFRESH_INTERVAL / 5
|
||||
RPC_ID_LENGTH = 20
|
||||
PROTOCOL_VERSION = 1
|
||||
BOTTOM_OUT_LIMIT = 3
|
||||
MSG_SIZE_LIMIT = 1400
|
||||
|
||||
|
||||
def digest(data: bytes) -> bytes:
|
||||
h = HASH_CLASS()
|
||||
h.update(data)
|
||||
return h.digest()
|
||||
|
||||
|
||||
def generate_id(num=None) -> bytes:
|
||||
if num is not None:
|
||||
return digest(str(num).encode())
|
||||
else:
|
||||
return digest(os.urandom(32))
|
||||
|
||||
|
||||
def generate_rpc_id(num=None) -> bytes:
|
||||
return generate_id(num)[:RPC_ID_LENGTH]
|
||||
@@ -1,23 +0,0 @@
|
||||
class BaseKademliaException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DecodeError(BaseKademliaException):
|
||||
"""
|
||||
Should be raised by an C{Encoding} implementation if decode operation
|
||||
fails
|
||||
"""
|
||||
|
||||
|
||||
class BucketFull(BaseKademliaException):
|
||||
"""
|
||||
Raised when the bucket is full
|
||||
"""
|
||||
|
||||
|
||||
class RemoteException(BaseKademliaException):
|
||||
pass
|
||||
|
||||
|
||||
class TransportNotConnected(BaseKademliaException):
|
||||
pass
|
||||
@@ -1,260 +0,0 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
import socket
|
||||
from lbry.utils import resolve_host
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
from lbry.dht.protocol.iterative_find import IterativeNodeFinder, IterativeValueFinder
|
||||
from lbry.dht.protocol.protocol import KademliaProtocol
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.peer import PeerManager
|
||||
from lbry.dht.peer import KademliaPeer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Node:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
|
||||
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.RPC_TIMEOUT,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX,
|
||||
storage: typing.Optional['SQLiteStorage'] = None):
|
||||
self.loop = loop
|
||||
self.internal_udp_port = internal_udp_port
|
||||
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
|
||||
split_buckets_under_index)
|
||||
self.listening_port: asyncio.DatagramTransport = None
|
||||
self.joined = asyncio.Event(loop=self.loop)
|
||||
self._join_task: asyncio.Task = None
|
||||
self._refresh_task: asyncio.Task = None
|
||||
self._storage = storage
|
||||
|
||||
async def refresh_node(self, force_once=False):
|
||||
while True:
|
||||
# remove peers with expired blob announcements from the datastore
|
||||
self.protocol.data_store.removed_expired_peers()
|
||||
|
||||
total_peers: typing.List['KademliaPeer'] = []
|
||||
# add all peers in the routing table
|
||||
total_peers.extend(self.protocol.routing_table.get_peers())
|
||||
# add all the peers who have announced blobs to us
|
||||
total_peers.extend(self.protocol.data_store.get_storing_contacts())
|
||||
|
||||
# get ids falling in the midpoint of each bucket that hasn't been recently updated
|
||||
node_ids = self.protocol.routing_table.get_refresh_list(0, True)
|
||||
# if we have 3 or fewer populated buckets get two random ids in the range of each to try and
|
||||
# populate/split the buckets further
|
||||
buckets_with_contacts = self.protocol.routing_table.buckets_with_contacts()
|
||||
if buckets_with_contacts <= 3:
|
||||
for i in range(buckets_with_contacts):
|
||||
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
|
||||
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
|
||||
|
||||
if self.protocol.routing_table.get_peers():
|
||||
# if we have node ids to look up, perform the iterative search until we have k results
|
||||
while node_ids:
|
||||
peers = await self.peer_search(node_ids.pop())
|
||||
total_peers.extend(peers)
|
||||
else:
|
||||
if force_once:
|
||||
break
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.REFRESH_INTERVAL // 4, fut.set_result, None)
|
||||
await fut
|
||||
continue
|
||||
|
||||
# ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated
|
||||
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
|
||||
if to_ping:
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
|
||||
if self._storage:
|
||||
await self._storage.save_kademlia_peers(self.protocol.routing_table.get_peers())
|
||||
if force_once:
|
||||
break
|
||||
|
||||
fut = asyncio.Future(loop=self.loop)
|
||||
self.loop.call_later(constants.REFRESH_INTERVAL, fut.set_result, None)
|
||||
await fut
|
||||
|
||||
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
|
||||
hash_value = binascii.unhexlify(blob_hash.encode())
|
||||
assert len(hash_value) == constants.HASH_LENGTH
|
||||
peers = await self.peer_search(hash_value)
|
||||
|
||||
if not self.protocol.external_ip:
|
||||
raise Exception("Cannot determine external IP")
|
||||
log.debug("Store to %i peers", len(peers))
|
||||
for peer in peers:
|
||||
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
|
||||
stored_to_tup = await asyncio.gather(
|
||||
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
|
||||
)
|
||||
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
|
||||
if stored_to:
|
||||
log.debug(
|
||||
"Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
|
||||
len(stored_to), len(peers)
|
||||
)
|
||||
else:
|
||||
log.debug("Failed announcing %s, stored to 0 peers", blob_hash[:8])
|
||||
return stored_to
|
||||
|
||||
def stop(self) -> None:
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
if self._join_task:
|
||||
self._join_task.cancel()
|
||||
if self._refresh_task and not (self._refresh_task.done() or self._refresh_task.cancelled()):
|
||||
self._refresh_task.cancel()
|
||||
if self.protocol and self.protocol.ping_queue.running:
|
||||
self.protocol.ping_queue.stop()
|
||||
self.protocol.stop()
|
||||
if self.listening_port is not None:
|
||||
self.listening_port.close()
|
||||
self._join_task = None
|
||||
self.listening_port = None
|
||||
log.info("Stopped DHT node")
|
||||
|
||||
async def start_listening(self, interface: str = '0.0.0.0') -> None:
|
||||
if not self.listening_port:
|
||||
self.listening_port, _ = await self.loop.create_datagram_endpoint(
|
||||
lambda: self.protocol, (interface, self.internal_udp_port)
|
||||
)
|
||||
log.info("DHT node listening on UDP %s:%i", interface, self.internal_udp_port)
|
||||
self.protocol.start()
|
||||
else:
|
||||
log.warning("Already bound to port %s", self.listening_port)
|
||||
|
||||
async def join_network(self, interface: str = '0.0.0.0',
|
||||
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
def peers_from_urls(urls: typing.Optional[typing.List[typing.Tuple[bytes, str, int, int]]]):
|
||||
peer_addresses = []
|
||||
for node_id, address, udp_port, tcp_port in urls:
|
||||
if (node_id, address, udp_port, tcp_port) not in peer_addresses and \
|
||||
(address, udp_port) != (self.protocol.external_ip, self.protocol.udp_port):
|
||||
peer_addresses.append((node_id, address, udp_port, tcp_port))
|
||||
return [make_kademlia_peer(*peer_address) for peer_address in peer_addresses]
|
||||
|
||||
if not self.listening_port:
|
||||
await self.start_listening(interface)
|
||||
self.protocol.ping_queue.start()
|
||||
self._refresh_task = self.loop.create_task(self.refresh_node())
|
||||
|
||||
while True:
|
||||
if self.protocol.routing_table.get_peers():
|
||||
if not self.joined.is_set():
|
||||
self.joined.set()
|
||||
log.info(
|
||||
"joined dht, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
|
||||
self.protocol.routing_table.buckets_with_contacts()
|
||||
)
|
||||
else:
|
||||
if self.joined.is_set():
|
||||
self.joined.clear()
|
||||
seed_peers = peers_from_urls(
|
||||
await self._storage.get_persisted_kademlia_peers()
|
||||
) if self._storage else []
|
||||
if not seed_peers:
|
||||
try:
|
||||
seed_peers.extend(peers_from_urls([
|
||||
(None, await resolve_host(address, udp_port, 'udp'), udp_port, None)
|
||||
for address, udp_port in known_node_urls or []
|
||||
]))
|
||||
except socket.gaierror:
|
||||
await asyncio.sleep(30, loop=self.loop)
|
||||
continue
|
||||
|
||||
self.protocol.peer_manager.reset()
|
||||
self.protocol.ping_queue.enqueue_maybe_ping(*seed_peers, delay=0.0)
|
||||
await self.peer_search(self.protocol.node_id, shortlist=seed_peers, count=32)
|
||||
|
||||
await asyncio.sleep(1, loop=self.loop)
|
||||
|
||||
def start(self, interface: str, known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
|
||||
self._join_task = self.loop.create_task(self.join_network(interface, known_node_urls))
|
||||
|
||||
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = constants.BOTTOM_OUT_LIMIT,
|
||||
max_results: int = constants.K) -> IterativeNodeFinder:
|
||||
|
||||
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
|
||||
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
|
||||
bottom_out_limit: int = 40,
|
||||
max_results: int = -1) -> IterativeValueFinder:
|
||||
|
||||
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
|
||||
key, bottom_out_limit, max_results, None, shortlist)
|
||||
|
||||
async def peer_search(self, node_id: bytes, count=constants.K, max_results=constants.K * 2,
|
||||
bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None
|
||||
) -> typing.List['KademliaPeer']:
|
||||
peers = []
|
||||
async for iteration_peers in self.get_iterative_node_finder(
|
||||
node_id, shortlist=shortlist, bottom_out_limit=bottom_out_limit, max_results=max_results):
|
||||
peers.extend(iteration_peers)
|
||||
distance = Distance(node_id)
|
||||
peers.sort(key=lambda peer: distance(peer.node_id))
|
||||
return peers[:count]
|
||||
|
||||
async def _accumulate_peers_for_value(self, search_queue: asyncio.Queue, result_queue: asyncio.Queue):
|
||||
tasks = []
|
||||
try:
|
||||
while True:
|
||||
blob_hash = await search_queue.get()
|
||||
tasks.append(self.loop.create_task(self._peers_for_value_producer(blob_hash, result_queue)))
|
||||
finally:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
|
||||
async def _peers_for_value_producer(self, blob_hash: str, result_queue: asyncio.Queue):
|
||||
async def put_into_result_queue_after_pong(_peer):
|
||||
try:
|
||||
await self.protocol.get_rpc_peer(_peer).ping()
|
||||
result_queue.put_nowait([_peer])
|
||||
log.debug("pong from %s:%i for %s", _peer.address, _peer.udp_port, blob_hash)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
# prioritize peers who reply to a dht ping first
|
||||
# this minimizes attempting to make tcp connections that won't work later to dead or unreachable peers
|
||||
|
||||
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
|
||||
to_put = []
|
||||
for peer in results:
|
||||
if peer.address == self.protocol.external_ip and self.protocol.peer_port == peer.tcp_port:
|
||||
continue
|
||||
is_good = self.protocol.peer_manager.peer_is_good(peer)
|
||||
if is_good:
|
||||
# the peer has replied recently over UDP, it can probably be reached on the TCP port
|
||||
to_put.append(peer)
|
||||
elif is_good is None:
|
||||
if not peer.udp_port:
|
||||
# TODO: use the same port for TCP and UDP
|
||||
# the udp port must be guessed
|
||||
# default to the ports being the same. if the TCP port appears to be <=0.48.0 default,
|
||||
# including on a network with several nodes, then assume the udp port is proportionately
|
||||
# based on a starting port of 4444
|
||||
udp_port_to_try = peer.tcp_port
|
||||
if 3400 > peer.tcp_port > 3332:
|
||||
udp_port_to_try = (peer.tcp_port - 3333) + 4444
|
||||
self.loop.create_task(put_into_result_queue_after_pong(
|
||||
make_kademlia_peer(peer.node_id, peer.address, udp_port_to_try, peer.tcp_port)
|
||||
))
|
||||
else:
|
||||
self.loop.create_task(put_into_result_queue_after_pong(peer))
|
||||
else:
|
||||
# the peer is known to be bad/unreachable, skip trying to connect to it over TCP
|
||||
log.debug("skip bad peer %s:%i for %s", peer.address, peer.tcp_port, blob_hash)
|
||||
if to_put:
|
||||
result_queue.put_nowait(to_put)
|
||||
|
||||
def accumulate_peers(self, search_queue: asyncio.Queue,
|
||||
peer_queue: typing.Optional[asyncio.Queue] = None
|
||||
) -> typing.Tuple[asyncio.Queue, asyncio.Task]:
|
||||
queue = peer_queue or asyncio.Queue(loop=self.loop)
|
||||
return queue, self.loop.create_task(self._accumulate_peers_for_value(search_queue, queue))
|
||||
@@ -1,196 +0,0 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache(1024)
|
||||
def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional[str],
|
||||
udp_port: typing.Optional[int] = None,
|
||||
tcp_port: typing.Optional[int] = None,
|
||||
allow_localhost: bool = False) -> 'KademliaPeer':
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
|
||||
|
||||
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
|
||||
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
|
||||
parsed_ip.is_reserved,
|
||||
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
|
||||
|
||||
class PeerManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop):
|
||||
self._loop = loop
|
||||
self._rpc_failures: typing.Dict[
|
||||
typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
|
||||
] = {}
|
||||
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {}
|
||||
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
|
||||
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
|
||||
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
|
||||
|
||||
def reset(self):
|
||||
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
|
||||
statistic.clear()
|
||||
|
||||
def report_failure(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
|
||||
self._rpc_failures[(address, udp_port)] = (previous, now)
|
||||
|
||||
def report_last_sent(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_sent[(address, udp_port)] = now
|
||||
|
||||
def report_last_replied(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_replied[(address, udp_port)] = now
|
||||
|
||||
def report_last_requested(self, address: str, udp_port: int):
|
||||
now = self._loop.time()
|
||||
self._last_requested[(address, udp_port)] = now
|
||||
|
||||
def clear_token(self, node_id: bytes):
|
||||
self._node_tokens.pop(node_id, None)
|
||||
|
||||
def update_token(self, node_id: bytes, token: bytes):
|
||||
now = self._loop.time()
|
||||
self._node_tokens[node_id] = (now, token)
|
||||
|
||||
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
|
||||
ts, token = self._node_tokens.get(node_id, (0, None))
|
||||
if ts and ts > self._loop.time() - constants.TOKEN_SECRET_REFRESH_INTERVAL:
|
||||
return token
|
||||
|
||||
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
|
||||
return self._last_replied.get((address, udp_port))
|
||||
|
||||
def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
|
||||
"""
|
||||
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
|
||||
This is to handle peers changing addresses and ids while assuring that the we only ever have
|
||||
one node id / address tuple mapped to each other
|
||||
"""
|
||||
if (address, udp_port) in self._node_id_mapping:
|
||||
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
|
||||
if node_id in self._node_id_reverse_mapping:
|
||||
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
|
||||
self._node_id_mapping[(address, udp_port)] = node_id
|
||||
self._node_id_reverse_mapping[node_id] = (address, udp_port)
|
||||
|
||||
def prune(self): # TODO: periodically call this
|
||||
now = self._loop.time()
|
||||
to_pop = []
|
||||
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
|
||||
if last_failure and last_failure < now - constants.RPC_ATTEMPTS_PRUNING_WINDOW:
|
||||
to_pop.append((address, udp_port))
|
||||
while to_pop:
|
||||
del self._rpc_failures[to_pop.pop()]
|
||||
to_pop = []
|
||||
for node_id, (age, token) in self._node_tokens.items(): # pylint: disable=unused-variable
|
||||
if age < now - constants.TOKEN_SECRET_REFRESH_INTERVAL:
|
||||
to_pop.append(node_id)
|
||||
while to_pop:
|
||||
del self._node_tokens[to_pop.pop()]
|
||||
|
||||
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int): # pylint: disable=too-many-return-statements
|
||||
"""
|
||||
:return: False if peer is bad, None if peer is unknown, or True if peer is good
|
||||
"""
|
||||
|
||||
delay = self._loop.time() - constants.CHECK_REFRESH_INTERVAL
|
||||
|
||||
# fixme: find a way to re-enable that without breaking other parts
|
||||
# if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
|
||||
# return
|
||||
# addr_tup = (address, udp_port)
|
||||
# if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
|
||||
# return
|
||||
previous_failure, most_recent_failure = self._rpc_failures.get((address, udp_port), (None, None))
|
||||
last_requested = self._last_requested.get((address, udp_port))
|
||||
last_replied = self._last_replied.get((address, udp_port))
|
||||
if node_id is None:
|
||||
return None
|
||||
if most_recent_failure and last_replied:
|
||||
if delay < last_replied > most_recent_failure:
|
||||
return True
|
||||
elif last_replied > most_recent_failure:
|
||||
return
|
||||
return False
|
||||
elif previous_failure and most_recent_failure and most_recent_failure > delay:
|
||||
return False
|
||||
elif last_replied and last_replied > delay:
|
||||
return True
|
||||
elif last_requested and last_requested > delay:
|
||||
return None
|
||||
return
|
||||
|
||||
def peer_is_good(self, peer: 'KademliaPeer'):
|
||||
return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port)
|
||||
|
||||
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer': # pylint: disable=no-self-use
|
||||
node_id, address, tcp_port = decode_compact_address(compact_address)
|
||||
return make_kademlia_peer(node_id, address, udp_port=None, tcp_port=tcp_port)
|
||||
|
||||
|
||||
@dataclass(unsafe_hash=True)
|
||||
class KademliaPeer:
|
||||
address: str = field(hash=True)
|
||||
_node_id: typing.Optional[bytes] = field(hash=True)
|
||||
udp_port: typing.Optional[int] = field(hash=True)
|
||||
tcp_port: typing.Optional[int] = field(compare=False, hash=False)
|
||||
protocol_version: typing.Optional[int] = field(default=1, compare=False, hash=False)
|
||||
allow_localhost: bool = field(default=False, compare=False, hash=False)
|
||||
|
||||
def __post_init__(self):
|
||||
if self._node_id is not None:
|
||||
if not len(self._node_id) == constants.HASH_LENGTH:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
|
||||
if self.udp_port is not None and not 1 <= self.udp_port <= 65535:
|
||||
raise ValueError("invalid udp port")
|
||||
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
|
||||
raise ValueError("invalid tcp port")
|
||||
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||
|
||||
def update_tcp_port(self, tcp_port: int):
|
||||
self.tcp_port = tcp_port
|
||||
|
||||
@property
|
||||
def node_id(self) -> bytes:
|
||||
return self._node_id
|
||||
|
||||
def compact_address_udp(self) -> bytearray:
|
||||
return make_compact_address(self.node_id, self.address, self.udp_port)
|
||||
|
||||
def compact_address_tcp(self) -> bytearray:
|
||||
return make_compact_address(self.node_id, self.address, self.tcp_port)
|
||||
|
||||
def compact_ip(self):
|
||||
return make_compact_ip(self.address)
|
||||
@@ -1,70 +0,0 @@
|
||||
import asyncio
|
||||
import typing
|
||||
|
||||
from lbry.dht import constants
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
|
||||
class DictDataStore:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager'):
|
||||
# Dictionary format:
|
||||
# { <key>: [(<contact>, <age>), ...] }
|
||||
self._data_store: typing.Dict[bytes, typing.List[typing.Tuple['KademliaPeer', float]]] = {}
|
||||
|
||||
self.loop = loop
|
||||
self._peer_manager = peer_manager
|
||||
self.completed_blobs: typing.Set[str] = set()
|
||||
|
||||
def removed_expired_peers(self):
|
||||
now = self.loop.time()
|
||||
keys = list(self._data_store.keys())
|
||||
for key in keys:
|
||||
to_remove = []
|
||||
for (peer, ts) in self._data_store[key]:
|
||||
if ts + constants.DATA_EXPIRATION < now or self._peer_manager.peer_is_good(peer) is False:
|
||||
to_remove.append((peer, ts))
|
||||
for item in to_remove:
|
||||
self._data_store[key].remove(item)
|
||||
if not self._data_store[key]:
|
||||
del self._data_store[key]
|
||||
|
||||
def filter_bad_and_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
|
||||
"""
|
||||
Returns only non-expired and unknown/good peers
|
||||
"""
|
||||
for peer in self.filter_expired_peers(key):
|
||||
if self._peer_manager.peer_is_good(peer) is not False:
|
||||
yield peer
|
||||
|
||||
def filter_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
|
||||
"""
|
||||
Returns only non-expired peers
|
||||
"""
|
||||
now = self.loop.time()
|
||||
for (peer, ts) in self._data_store.get(key, []):
|
||||
if ts + constants.DATA_EXPIRATION > now:
|
||||
yield peer
|
||||
|
||||
def has_peers_for_blob(self, key: bytes) -> bool:
|
||||
return key in self._data_store
|
||||
|
||||
def add_peer_to_blob(self, contact: 'KademliaPeer', key: bytes) -> None:
|
||||
now = self.loop.time()
|
||||
if key in self._data_store:
|
||||
current = list(filter(lambda x: x[0] == contact, self._data_store[key]))
|
||||
if len(current) > 0:
|
||||
self._data_store[key][self._data_store[key].index(current[0])] = contact, now
|
||||
else:
|
||||
self._data_store[key].append((contact, now))
|
||||
else:
|
||||
self._data_store[key] = [(contact, now)]
|
||||
|
||||
def get_peers_for_blob(self, key: bytes) -> typing.List['KademliaPeer']:
|
||||
return list(self.filter_bad_and_expired_peers(key))
|
||||
|
||||
def get_storing_contacts(self) -> typing.List['KademliaPeer']:
|
||||
peers = set()
|
||||
for _, stored in self._data_store.items():
|
||||
peers.update(set(map(lambda tup: tup[0], stored)))
|
||||
return list(peers)
|
||||
@@ -1,25 +0,0 @@
|
||||
from lbry.dht import constants
|
||||
|
||||
|
||||
class Distance:
|
||||
"""Calculate the XOR result between two string variables.
|
||||
|
||||
Frequently we re-use one of the points so as an optimization
|
||||
we pre-calculate the value of that point.
|
||||
"""
|
||||
|
||||
def __init__(self, key: bytes):
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError(f"invalid key length: {len(key)}")
|
||||
self.key = key
|
||||
self.val_key_one = int.from_bytes(key, 'big')
|
||||
|
||||
def __call__(self, key_two: bytes) -> int:
|
||||
if len(key_two) != constants.HASH_LENGTH:
|
||||
raise ValueError(f"invalid length of key to compare: {len(key_two)}")
|
||||
val_key_two = int.from_bytes(key_two, 'big')
|
||||
return self.val_key_one ^ val_key_two
|
||||
|
||||
def is_closer(self, key_a: bytes, key_b: bytes) -> bool:
|
||||
"""Returns true is `key_a` is closer to `key` than `key_b` is"""
|
||||
return self(key_a) < self(key_b)
|
||||
@@ -1,385 +0,0 @@
|
||||
import asyncio
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
from collections import defaultdict
|
||||
import typing
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
from lbry.dht.serialization.datagram import PAGE_KEY
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from lbry.dht.protocol.routing_table import TreeRoutingTable
|
||||
from lbry.dht.protocol.protocol import KademliaProtocol
|
||||
from lbry.dht.peer import PeerManager, KademliaPeer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FindResponse:
|
||||
@property
|
||||
def found(self) -> bool:
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class FindNodeResponse(FindResponse):
|
||||
def __init__(self, key: bytes, close_triples: typing.List[typing.Tuple[bytes, str, int]]):
|
||||
self.key = key
|
||||
self.close_triples = close_triples
|
||||
|
||||
@property
|
||||
def found(self) -> bool:
|
||||
return self.key in [triple[0] for triple in self.close_triples]
|
||||
|
||||
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
return self.close_triples
|
||||
|
||||
|
||||
class FindValueResponse(FindResponse):
|
||||
def __init__(self, key: bytes, result_dict: typing.Dict):
|
||||
self.key = key
|
||||
self.token = result_dict[b'token']
|
||||
self.close_triples: typing.List[typing.Tuple[bytes, bytes, int]] = result_dict.get(b'contacts', [])
|
||||
self.found_compact_addresses = result_dict.get(key, [])
|
||||
self.pages = int(result_dict.get(PAGE_KEY, 0))
|
||||
|
||||
@property
|
||||
def found(self) -> bool:
|
||||
return len(self.found_compact_addresses) > 0
|
||||
|
||||
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
return [(node_id, address.decode(), port) for node_id, address, port in self.close_triples]
|
||||
|
||||
|
||||
def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']]) -> typing.List['KademliaPeer']:
|
||||
"""
|
||||
If not provided, initialize the shortlist of peers to probe to the (up to) k closest peers in the routing table
|
||||
|
||||
:param routing_table: a TreeRoutingTable
|
||||
:param key: a 48 byte hash
|
||||
:param shortlist: optional manually provided shortlist, this is done during bootstrapping when there are no
|
||||
peers in the routing table. During bootstrap the shortlist is set to be the seed nodes.
|
||||
"""
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
return shortlist or routing_table.find_close_peers(key)
|
||||
|
||||
|
||||
class IterativeFinder:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError("invalid key length: %i" % len(key))
|
||||
self.loop = loop
|
||||
self.peer_manager = peer_manager
|
||||
self.routing_table = routing_table
|
||||
self.protocol = protocol
|
||||
|
||||
self.key = key
|
||||
self.bottom_out_limit = bottom_out_limit
|
||||
self.max_results = max_results
|
||||
self.exclude = exclude or []
|
||||
|
||||
self.active: typing.Set['KademliaPeer'] = set()
|
||||
self.contacted: typing.Set['KademliaPeer'] = set()
|
||||
self.distance = Distance(key)
|
||||
|
||||
self.closest_peer: typing.Optional['KademliaPeer'] = None
|
||||
self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
|
||||
|
||||
self.iteration_queue = asyncio.Queue(loop=self.loop)
|
||||
|
||||
self.running_probes: typing.Set[asyncio.Task] = set()
|
||||
self.iteration_count = 0
|
||||
self.bottom_out_count = 0
|
||||
self.running = False
|
||||
self.tasks: typing.List[asyncio.Task] = []
|
||||
self.delayed_calls: typing.List[asyncio.Handle] = []
|
||||
for peer in get_shortlist(routing_table, key, shortlist):
|
||||
if peer.node_id:
|
||||
self._add_active(peer)
|
||||
else:
|
||||
# seed nodes
|
||||
self._schedule_probe(peer)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
|
||||
"""
|
||||
Send the rpc request to the peer and return an object with the FindResponse interface
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def search_exhausted(self):
|
||||
"""
|
||||
This method ends the iterator due no more peers to contact.
|
||||
Override to provide last time results.
|
||||
"""
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindResponse):
|
||||
"""
|
||||
Called after adding peers from an rpc result to the shortlist.
|
||||
This method is responsible for putting a result for the generator into the Queue
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']: #pylint: disable=no-self-use
|
||||
"""
|
||||
Get an initial or cached result to be put into the Queue. Used for findValue requests where the blob
|
||||
has peers in the local data store of blobs announced to us
|
||||
"""
|
||||
return []
|
||||
|
||||
def _is_closer(self, peer: 'KademliaPeer') -> bool:
|
||||
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
|
||||
|
||||
def _add_active(self, peer):
|
||||
if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id:
|
||||
self.active.add(peer)
|
||||
if self._is_closer(peer):
|
||||
self.prev_closest_peer = self.closest_peer
|
||||
self.closest_peer = peer
|
||||
|
||||
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
|
||||
self._add_active(peer)
|
||||
for contact_triple in response.get_close_triples():
|
||||
node_id, address, udp_port = contact_triple
|
||||
try:
|
||||
self._add_active(make_kademlia_peer(node_id, address, udp_port))
|
||||
except ValueError:
|
||||
log.warning("misbehaving peer %s:%i returned peer with reserved ip %s:%i", peer.address,
|
||||
peer.udp_port, address, udp_port)
|
||||
self.check_result_ready(response)
|
||||
|
||||
async def _send_probe(self, peer: 'KademliaPeer'):
|
||||
try:
|
||||
response = await self.send_probe(peer)
|
||||
except asyncio.TimeoutError:
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except ValueError as err:
|
||||
log.warning(str(err))
|
||||
self.active.discard(peer)
|
||||
return
|
||||
except TransportNotConnected:
|
||||
return self.aclose()
|
||||
except RemoteException:
|
||||
return
|
||||
return await self._handle_probe_result(peer, response)
|
||||
|
||||
async def _search_round(self):
|
||||
"""
|
||||
Send up to constants.alpha (5) probes to closest active peers
|
||||
"""
|
||||
|
||||
added = 0
|
||||
to_probe = list(self.active - self.contacted)
|
||||
to_probe.sort(key=lambda peer: self.distance(self.key))
|
||||
for peer in to_probe:
|
||||
if added >= constants.ALPHA:
|
||||
break
|
||||
origin_address = (peer.address, peer.udp_port)
|
||||
if origin_address in self.exclude:
|
||||
continue
|
||||
if peer.node_id == self.protocol.node_id:
|
||||
continue
|
||||
if origin_address == (self.protocol.external_ip, self.protocol.udp_port):
|
||||
continue
|
||||
self._schedule_probe(peer)
|
||||
added += 1
|
||||
log.debug("running %d probes", len(self.running_probes))
|
||||
if not added and not self.running_probes:
|
||||
log.debug("search for %s exhausted", hexlify(self.key)[:8])
|
||||
self.search_exhausted()
|
||||
|
||||
def _schedule_probe(self, peer: 'KademliaPeer'):
|
||||
self.contacted.add(peer)
|
||||
|
||||
t = self.loop.create_task(self._send_probe(peer))
|
||||
|
||||
def callback(_):
|
||||
self.running_probes.difference_update({
|
||||
probe for probe in self.running_probes if probe.done() or probe == t
|
||||
})
|
||||
if not self.running_probes:
|
||||
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
|
||||
|
||||
t.add_done_callback(callback)
|
||||
self.running_probes.add(t)
|
||||
|
||||
async def _search_task(self, delay: typing.Optional[float] = constants.ITERATIVE_LOOKUP_DELAY):
|
||||
try:
|
||||
if self.running:
|
||||
await self._search_round()
|
||||
if self.running:
|
||||
self.delayed_calls.append(self.loop.call_later(delay, self._search))
|
||||
except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
|
||||
if self.running:
|
||||
self.loop.call_soon(self.aclose)
|
||||
|
||||
def _search(self):
|
||||
self.tasks.append(self.loop.create_task(self._search_task()))
|
||||
|
||||
def __aiter__(self):
|
||||
if self.running:
|
||||
raise Exception("already running")
|
||||
self.running = True
|
||||
self._search()
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> typing.List['KademliaPeer']:
|
||||
try:
|
||||
if self.iteration_count == 0:
|
||||
result = self.get_initial_result() or await self.iteration_queue.get()
|
||||
else:
|
||||
result = await self.iteration_queue.get()
|
||||
if not result:
|
||||
raise StopAsyncIteration
|
||||
self.iteration_count += 1
|
||||
return result
|
||||
except (asyncio.CancelledError, StopAsyncIteration):
|
||||
self.loop.call_soon(self.aclose)
|
||||
raise
|
||||
|
||||
def aclose(self):
|
||||
self.running = False
|
||||
self.iteration_queue.put_nowait(None)
|
||||
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
|
||||
task.cancel()
|
||||
self.tasks.clear()
|
||||
self.running_probes.clear()
|
||||
self.delayed_calls.clear()
|
||||
|
||||
|
||||
class IterativeNodeFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
shortlist)
|
||||
self.yielded_peers: typing.Set['KademliaPeer'] = set()
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
|
||||
log.debug("probing %s:%d %s", peer.address, peer.udp_port, hexlify(peer.node_id)[:8] if peer.node_id else '')
|
||||
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
|
||||
return FindNodeResponse(self.key, response)
|
||||
|
||||
def search_exhausted(self):
|
||||
self.put_result(self.active, finish=True)
|
||||
|
||||
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
|
||||
not_yet_yielded = [
|
||||
peer for peer in from_iter
|
||||
if peer not in self.yielded_peers
|
||||
and peer.node_id != self.protocol.node_id
|
||||
and self.peer_manager.peer_is_good(peer) is not False
|
||||
]
|
||||
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
|
||||
to_yield = not_yet_yielded[:min(constants.K, len(not_yet_yielded))]
|
||||
if to_yield:
|
||||
self.yielded_peers.update(to_yield)
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
if finish:
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def check_result_ready(self, response: FindNodeResponse):
|
||||
found = response.found and self.key != self.protocol.node_id
|
||||
|
||||
if found:
|
||||
log.debug("found")
|
||||
return self.put_result(self.active, finish=True)
|
||||
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
|
||||
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
|
||||
# self.bottom_out_count, self.iteration_count)
|
||||
self.bottom_out_count = 0
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
log.info("bottom out %i %i %i", len(self.active), len(self.contacted), self.bottom_out_count)
|
||||
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
|
||||
log.info("limit hit")
|
||||
self.put_result(self.active, True)
|
||||
|
||||
|
||||
class IterativeValueFinder(IterativeFinder):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager',
|
||||
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
|
||||
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.K,
|
||||
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
|
||||
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
|
||||
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
|
||||
shortlist)
|
||||
self.blob_peers: typing.Set['KademliaPeer'] = set()
|
||||
# this tracks the index of the most recent page we requested from each peer
|
||||
self.peer_pages: typing.DefaultDict['KademliaPeer', int] = defaultdict(int)
|
||||
# this tracks the set of blob peers returned by each peer
|
||||
self.discovered_peers: typing.Dict['KademliaPeer', typing.Set['KademliaPeer']] = defaultdict(set)
|
||||
|
||||
async def send_probe(self, peer: 'KademliaPeer') -> FindValueResponse:
|
||||
page = self.peer_pages[peer]
|
||||
response = await self.protocol.get_rpc_peer(peer).find_value(self.key, page=page)
|
||||
parsed = FindValueResponse(self.key, response)
|
||||
if not parsed.found:
|
||||
return parsed
|
||||
already_known = len(self.discovered_peers[peer])
|
||||
decoded_peers = set()
|
||||
for compact_addr in parsed.found_compact_addresses:
|
||||
try:
|
||||
decoded_peers.add(self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr))
|
||||
except ValueError:
|
||||
log.warning("misbehaving peer %s:%i returned invalid peer for blob",
|
||||
peer.address, peer.udp_port)
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
parsed.found_compact_addresses.clear()
|
||||
return parsed
|
||||
self.discovered_peers[peer].update(decoded_peers)
|
||||
log.debug("probed %s:%i page %i, %i known", peer.address, peer.udp_port, page,
|
||||
already_known + len(parsed.found_compact_addresses))
|
||||
if len(self.discovered_peers[peer]) != already_known + len(parsed.found_compact_addresses):
|
||||
log.warning("misbehaving peer %s:%i returned duplicate peers for blob", peer.address, peer.udp_port)
|
||||
parsed.found_compact_addresses.clear()
|
||||
elif len(parsed.found_compact_addresses) >= constants.K and self.peer_pages[peer] < parsed.pages:
|
||||
# the peer returned a full page and indicates it has more
|
||||
self.peer_pages[peer] += 1
|
||||
if peer in self.contacted:
|
||||
# the peer must be removed from self.contacted so that it will be probed for the next page
|
||||
self.contacted.remove(peer)
|
||||
return parsed
|
||||
|
||||
def check_result_ready(self, response: FindValueResponse):
|
||||
if response.found:
|
||||
blob_peers = [self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr)
|
||||
for compact_addr in response.found_compact_addresses]
|
||||
to_yield = []
|
||||
self.bottom_out_count = 0
|
||||
for blob_peer in blob_peers:
|
||||
if blob_peer not in self.blob_peers:
|
||||
self.blob_peers.add(blob_peer)
|
||||
to_yield.append(blob_peer)
|
||||
if to_yield:
|
||||
# log.info("found %i new peers for blob", len(to_yield))
|
||||
self.iteration_queue.put_nowait(to_yield)
|
||||
# if self.max_results and len(self.blob_peers) >= self.max_results:
|
||||
# log.info("enough blob peers found")
|
||||
# if not self.finished.is_set():
|
||||
# self.finished.set()
|
||||
elif self.prev_closest_peer and self.closest_peer:
|
||||
self.bottom_out_count += 1
|
||||
if self.bottom_out_count >= self.bottom_out_limit:
|
||||
log.info("blob peer search bottomed out")
|
||||
self.iteration_queue.put_nowait(None)
|
||||
|
||||
def get_initial_result(self) -> typing.List['KademliaPeer']:
|
||||
if self.protocol.data_store.has_peers_for_blob(self.key):
|
||||
return self.protocol.data_store.get_peers_for_blob(self.key)
|
||||
return []
|
||||
@@ -1,682 +0,0 @@
|
||||
import logging
|
||||
import socket
|
||||
import functools
|
||||
import hashlib
|
||||
import asyncio
|
||||
import typing
|
||||
import binascii
|
||||
import random
|
||||
from asyncio.protocols import DatagramProtocol
|
||||
from asyncio.transports import DatagramTransport
|
||||
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
|
||||
from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
|
||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||
from lbry.dht.protocol.routing_table import TreeRoutingTable
|
||||
from lbry.dht.protocol.data_store import DictDataStore
|
||||
from lbry.dht.peer import make_kademlia_peer
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.peer import PeerManager, KademliaPeer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
OLD_PROTOCOL_ERRORS = {
|
||||
"findNode() takes exactly 2 arguments (5 given)": "0.19.1",
|
||||
"findValue() takes exactly 2 arguments (5 given)": "0.19.1"
|
||||
}
|
||||
|
||||
|
||||
class KademliaRPC:
|
||||
def __init__(self, protocol: 'KademliaProtocol', loop: asyncio.AbstractEventLoop, peer_port: int = 3333):
|
||||
self.protocol = protocol
|
||||
self.loop = loop
|
||||
self.peer_port = peer_port
|
||||
self.old_token_secret: bytes = None
|
||||
self.token_secret = constants.generate_id()
|
||||
|
||||
def compact_address(self):
|
||||
compact_ip = functools.reduce(lambda buff, x: buff + bytearray([int(x)]),
|
||||
self.protocol.external_ip.split('.'), bytearray())
|
||||
compact_port = self.peer_port.to_bytes(2, 'big')
|
||||
return compact_ip + compact_port + self.protocol.node_id
|
||||
|
||||
@staticmethod
|
||||
def ping():
|
||||
return b'pong'
|
||||
|
||||
def store(self, rpc_contact: 'KademliaPeer', blob_hash: bytes, token: bytes, port: int) -> bytes:
|
||||
if len(blob_hash) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid length of blob hash: {len(blob_hash)}")
|
||||
if not 0 < port < 65535:
|
||||
raise ValueError(f"invalid tcp port: {port}")
|
||||
rpc_contact.update_tcp_port(port)
|
||||
if not self.verify_token(token, rpc_contact.compact_ip()):
|
||||
if self.loop.time() - self.protocol.started_listening_time < constants.TOKEN_SECRET_REFRESH_INTERVAL:
|
||||
pass
|
||||
else:
|
||||
raise ValueError("Invalid token")
|
||||
self.protocol.data_store.add_peer_to_blob(
|
||||
rpc_contact, blob_hash
|
||||
)
|
||||
return b'OK'
|
||||
|
||||
def find_node(self, rpc_contact: 'KademliaPeer', key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError("invalid contact node_id length: %i" % len(key))
|
||||
|
||||
contacts = self.protocol.routing_table.find_close_peers(key, sender_node_id=rpc_contact.node_id)
|
||||
contact_triples = []
|
||||
for contact in contacts[:constants.K * 2]:
|
||||
contact_triples.append((contact.node_id, contact.address, contact.udp_port))
|
||||
return contact_triples
|
||||
|
||||
def find_value(self, rpc_contact: 'KademliaPeer', key: bytes, page: int = 0):
|
||||
page = page if page > 0 else 0
|
||||
|
||||
if len(key) != constants.HASH_LENGTH:
|
||||
raise ValueError("invalid blob_exchange hash length: %i" % len(key))
|
||||
|
||||
response = {
|
||||
b'token': self.make_token(rpc_contact.compact_ip()),
|
||||
}
|
||||
|
||||
if not page:
|
||||
response[b'contacts'] = self.find_node(rpc_contact, key)[:constants.K]
|
||||
|
||||
if self.protocol.protocol_version:
|
||||
response[b'protocolVersion'] = self.protocol.protocol_version
|
||||
|
||||
# get peers we have stored for this blob_exchange
|
||||
peers = [
|
||||
peer.compact_address_tcp()
|
||||
for peer in self.protocol.data_store.get_peers_for_blob(key)
|
||||
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
|
||||
]
|
||||
# if we don't have k storing peers to return and we have this hash locally, include our contact information
|
||||
if len(peers) < constants.K and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
|
||||
peers.append(self.compact_address())
|
||||
if not peers:
|
||||
response[PAGE_KEY] = 0
|
||||
else:
|
||||
response[PAGE_KEY] = (len(peers) // (constants.K + 1)) + 1 # how many pages of peers we have for the blob
|
||||
if len(peers) > constants.K:
|
||||
random.Random(self.protocol.node_id).shuffle(peers)
|
||||
if page * constants.K < len(peers):
|
||||
response[key] = peers[page * constants.K:page * constants.K + constants.K]
|
||||
return response
|
||||
|
||||
def refresh_token(self): # TODO: this needs to be called periodically
|
||||
self.old_token_secret = self.token_secret
|
||||
self.token_secret = constants.generate_id()
|
||||
|
||||
def make_token(self, compact_ip):
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.token_secret + compact_ip)
|
||||
return h.digest()
|
||||
|
||||
def verify_token(self, token, compact_ip):
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.token_secret + compact_ip)
|
||||
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
|
||||
h = hashlib.new('sha384')
|
||||
h.update(self.old_token_secret + compact_ip)
|
||||
if not token == h.digest():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class RemoteKademliaRPC:
|
||||
"""
|
||||
Encapsulates RPC calls to remote Peers
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_tracker: 'PeerManager', protocol: 'KademliaProtocol',
|
||||
peer: 'KademliaPeer'):
|
||||
self.loop = loop
|
||||
self.peer_tracker = peer_tracker
|
||||
self.protocol = protocol
|
||||
self.peer = peer
|
||||
|
||||
async def ping(self) -> bytes:
|
||||
"""
|
||||
:return: b'pong'
|
||||
"""
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_ping(self.protocol.node_id)
|
||||
)
|
||||
return response.response
|
||||
|
||||
async def store(self, blob_hash: bytes) -> bytes:
|
||||
"""
|
||||
:param blob_hash: blob hash as bytes
|
||||
:return: b'OK'
|
||||
"""
|
||||
if len(blob_hash) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid length of blob hash: {len(blob_hash)}")
|
||||
if not self.protocol.peer_port or not 0 < self.protocol.peer_port < 65535:
|
||||
raise ValueError(f"invalid tcp port: {self.protocol.peer_port}")
|
||||
token = self.peer_tracker.get_node_token(self.peer.node_id)
|
||||
if not token:
|
||||
find_value_resp = await self.find_value(blob_hash)
|
||||
token = find_value_resp[b'token']
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_store(self.protocol.node_id, blob_hash, token, self.protocol.peer_port)
|
||||
)
|
||||
return response.response
|
||||
|
||||
async def find_node(self, key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
|
||||
"""
|
||||
:return: [(node_id, address, udp_port), ...]
|
||||
"""
|
||||
if len(key) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid length of find node key: {len(key)}")
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_node(self.protocol.node_id, key)
|
||||
)
|
||||
return [(node_id, address.decode(), udp_port) for node_id, address, udp_port in response.response]
|
||||
|
||||
async def find_value(self, key: bytes, page: int = 0) -> typing.Union[typing.Dict]:
|
||||
"""
|
||||
:return: {
|
||||
b'token': <token bytes>,
|
||||
b'contacts': [(node_id, address, udp_port), ...]
|
||||
<key bytes>: [<blob_peer_compact_address, ...]
|
||||
}
|
||||
"""
|
||||
if len(key) != constants.HASH_BITS // 8:
|
||||
raise ValueError(f"invalid length of find value key: {len(key)}")
|
||||
response = await self.protocol.send_request(
|
||||
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key, page=page)
|
||||
)
|
||||
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
|
||||
return response.response
|
||||
|
||||
|
||||
class PingQueue:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, protocol: 'KademliaProtocol'):
|
||||
self._loop = loop
|
||||
self._protocol = protocol
|
||||
self._pending_contacts: typing.Dict['KademliaPeer', float] = {}
|
||||
self._process_task: asyncio.Task = None
|
||||
self._running = False
|
||||
self._running_pings: typing.Set[asyncio.Task] = set()
|
||||
self._default_delay = constants.MAYBE_PING_DELAY
|
||||
|
||||
@property
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
|
||||
delay = delay if delay is not None else self._default_delay
|
||||
now = self._loop.time()
|
||||
for peer in peers:
|
||||
if peer not in self._pending_contacts or now + delay < self._pending_contacts[peer]:
|
||||
self._pending_contacts[peer] = delay + now
|
||||
|
||||
def maybe_ping(self, peer: 'KademliaPeer'):
|
||||
async def ping_task():
|
||||
try:
|
||||
if self._protocol.peer_manager.peer_is_good(peer):
|
||||
if peer not in self._protocol.routing_table.get_peers():
|
||||
self._protocol.add_peer(peer)
|
||||
return
|
||||
await self._protocol.get_rpc_peer(peer).ping()
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
pass
|
||||
|
||||
task = self._loop.create_task(ping_task())
|
||||
task.add_done_callback(lambda _: None if task not in self._running_pings else self._running_pings.remove(task))
|
||||
self._running_pings.add(task)
|
||||
|
||||
async def _process(self): # send up to 1 ping per second
|
||||
while True:
|
||||
enqueued = list(self._pending_contacts.keys())
|
||||
now = self._loop.time()
|
||||
for peer in enqueued:
|
||||
if self._pending_contacts[peer] <= now:
|
||||
del self._pending_contacts[peer]
|
||||
self.maybe_ping(peer)
|
||||
break
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
def start(self):
|
||||
assert not self._running
|
||||
self._running = True
|
||||
if not self._process_task:
|
||||
self._process_task = self._loop.create_task(self._process())
|
||||
|
||||
def stop(self):
|
||||
assert self._running
|
||||
self._running = False
|
||||
if self._process_task:
|
||||
self._process_task.cancel()
|
||||
self._process_task = None
|
||||
while self._running_pings:
|
||||
self._running_pings.pop().cancel()
|
||||
|
||||
|
||||
class KademliaProtocol(DatagramProtocol):
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
|
||||
udp_port: int, peer_port: int, rpc_timeout: float = constants.RPC_TIMEOUT,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
|
||||
self.peer_manager = peer_manager
|
||||
self.loop = loop
|
||||
self.node_id = node_id
|
||||
self.external_ip = external_ip
|
||||
self.udp_port = udp_port
|
||||
self.peer_port = peer_port
|
||||
self.is_seed_node = False
|
||||
self.partial_messages: typing.Dict[bytes, typing.Dict[bytes, bytes]] = {}
|
||||
self.sent_messages: typing.Dict[bytes, typing.Tuple['KademliaPeer', asyncio.Future, RequestDatagram]] = {}
|
||||
self.protocol_version = constants.PROTOCOL_VERSION
|
||||
self.started_listening_time = 0
|
||||
self.transport: DatagramTransport = None
|
||||
self.old_token_secret = constants.generate_id()
|
||||
self.token_secret = constants.generate_id()
|
||||
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index)
|
||||
self.data_store = DictDataStore(self.loop, self.peer_manager)
|
||||
self.ping_queue = PingQueue(self.loop, self)
|
||||
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
|
||||
self.rpc_timeout = rpc_timeout
|
||||
self._split_lock = asyncio.Lock(loop=self.loop)
|
||||
self._to_remove: typing.Set['KademliaPeer'] = set()
|
||||
self._to_add: typing.Set['KademliaPeer'] = set()
|
||||
self._wakeup_routing_task = asyncio.Event(loop=self.loop)
|
||||
self.maintaing_routing_task: typing.Optional[asyncio.Task] = None
|
||||
|
||||
@functools.lru_cache(128)
|
||||
def get_rpc_peer(self, peer: 'KademliaPeer') -> RemoteKademliaRPC:
|
||||
return RemoteKademliaRPC(self.loop, self.peer_manager, self, peer)
|
||||
|
||||
def start(self):
|
||||
self.maintaing_routing_task = self.loop.create_task(self.routing_table_task())
|
||||
|
||||
def stop(self):
|
||||
if self.maintaing_routing_task:
|
||||
self.maintaing_routing_task.cancel()
|
||||
if self.transport:
|
||||
self.disconnect()
|
||||
|
||||
def disconnect(self):
|
||||
self.transport.close()
|
||||
|
||||
def connection_made(self, transport: DatagramTransport):
|
||||
self.transport = transport
|
||||
|
||||
def connection_lost(self, exc):
|
||||
self.stop()
|
||||
|
||||
@staticmethod
|
||||
def _migrate_incoming_rpc_args(peer: 'KademliaPeer', method: bytes, *args) -> typing.Tuple[typing.Tuple,
|
||||
typing.Dict]:
|
||||
if method == b'store' and peer.protocol_version == 0:
|
||||
if isinstance(args[1], dict):
|
||||
blob_hash = args[0]
|
||||
token = args[1].pop(b'token', None)
|
||||
port = args[1].pop(b'port', -1)
|
||||
original_publisher_id = args[1].pop(b'lbryid', None)
|
||||
age = 0
|
||||
return (blob_hash, token, port, original_publisher_id, age), {}
|
||||
return args, {}
|
||||
|
||||
async def _add_peer(self, peer: 'KademliaPeer'):
|
||||
if not peer.node_id:
|
||||
log.warning("Tried adding a peer with no node id!")
|
||||
return False
|
||||
for my_peer in self.routing_table.get_peers():
|
||||
if (my_peer.address, my_peer.udp_port) == (peer.address, peer.udp_port) and my_peer.node_id != peer.node_id:
|
||||
self.routing_table.remove_peer(my_peer)
|
||||
self.routing_table.join_buckets()
|
||||
bucket_index = self.routing_table.kbucket_index(peer.node_id)
|
||||
if self.routing_table.buckets[bucket_index].add_peer(peer):
|
||||
return True
|
||||
|
||||
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
|
||||
if self.routing_table.should_split(bucket_index, peer.node_id):
|
||||
self.routing_table.split_bucket(bucket_index)
|
||||
# Retry the insertion attempt
|
||||
result = await self._add_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
return result
|
||||
else:
|
||||
# We can't split the k-bucket
|
||||
#
|
||||
# The 13 page kademlia paper specifies that the least recently contacted node in the bucket
|
||||
# shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
|
||||
# the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
|
||||
#
|
||||
# A reasonable extension to this is BEP 0005, which extends the above:
|
||||
#
|
||||
# Not all nodes that we learn about are equal. Some are "good" and some are not.
|
||||
# Many nodes using the DHT are able to send queries and receive responses,
|
||||
# but are not able to respond to queries from other nodes. It is important that
|
||||
# each node's routing table must contain only known good nodes. A good node is
|
||||
# a node has responded to one of our queries within the last 15 minutes. A node
|
||||
# is also good if it has ever responded to one of our queries and has sent us a
|
||||
# query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
|
||||
# questionable. Nodes become bad when they fail to respond to multiple queries
|
||||
# in a row. Nodes that we know are good are given priority over nodes with unknown status.
|
||||
#
|
||||
# When there are bad or questionable nodes in the bucket, the least recent is selected for
|
||||
# potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
|
||||
# contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
|
||||
# is ignored if the pinged node replies.
|
||||
|
||||
not_good_contacts = self.routing_table.buckets[bucket_index].get_bad_or_unknown_peers()
|
||||
not_recently_replied = []
|
||||
for my_peer in not_good_contacts:
|
||||
last_replied = self.peer_manager.get_last_replied(my_peer.address, my_peer.udp_port)
|
||||
if not last_replied or last_replied + 60 < self.loop.time():
|
||||
not_recently_replied.append(my_peer)
|
||||
if not_recently_replied:
|
||||
to_replace = not_recently_replied[0]
|
||||
else:
|
||||
to_replace = self.routing_table.buckets[bucket_index].peers[0]
|
||||
last_replied = self.peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
|
||||
if last_replied and last_replied + 60 > self.loop.time():
|
||||
return False
|
||||
log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
|
||||
try:
|
||||
to_replace_rpc = self.get_rpc_peer(to_replace)
|
||||
await to_replace_rpc.ping()
|
||||
return False
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
|
||||
to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
|
||||
if to_replace in self.routing_table.buckets[bucket_index]:
|
||||
self.routing_table.buckets[bucket_index].remove_peer(to_replace)
|
||||
return await self._add_peer(peer)
|
||||
|
||||
def add_peer(self, peer: 'KademliaPeer'):
|
||||
if peer.node_id == self.node_id:
|
||||
return False
|
||||
self._to_add.add(peer)
|
||||
self._wakeup_routing_task.set()
|
||||
|
||||
def remove_peer(self, peer: 'KademliaPeer'):
|
||||
self._to_remove.add(peer)
|
||||
self._wakeup_routing_task.set()
|
||||
|
||||
async def routing_table_task(self):
|
||||
while True:
|
||||
while self._to_remove:
|
||||
async with self._split_lock:
|
||||
peer = self._to_remove.pop()
|
||||
self.routing_table.remove_peer(peer)
|
||||
self.routing_table.join_buckets()
|
||||
while self._to_add:
|
||||
async with self._split_lock:
|
||||
await self._add_peer(self._to_add.pop())
|
||||
await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1, loop=self.loop), loop=self.loop)
|
||||
self._wakeup_routing_task.clear()
|
||||
|
||||
def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
|
||||
assert sender_contact.node_id != self.node_id, (binascii.hexlify(sender_contact.node_id)[:8].decode(),
|
||||
binascii.hexlify(self.node_id)[:8].decode())
|
||||
method = message.method
|
||||
if method not in [b'ping', b'store', b'findNode', b'findValue']:
|
||||
raise AttributeError('Invalid method: %s' % message.method.decode())
|
||||
if message.args and isinstance(message.args[-1], dict) and b'protocolVersion' in message.args[-1]:
|
||||
# args don't need reformatting
|
||||
args, kwargs = tuple(message.args[:-1]), message.args[-1]
|
||||
else:
|
||||
args, kwargs = self._migrate_incoming_rpc_args(sender_contact, message.method, *message.args)
|
||||
log.debug("%s:%i RECV CALL %s %s:%i", self.external_ip, self.udp_port, message.method.decode(),
|
||||
sender_contact.address, sender_contact.udp_port)
|
||||
|
||||
if method == b'ping':
|
||||
result = self.node_rpc.ping()
|
||||
elif method == b'store':
|
||||
blob_hash, token, port, original_publisher_id, age = args[:5] # pylint: disable=unused-variable
|
||||
result = self.node_rpc.store(sender_contact, blob_hash, token, port)
|
||||
else:
|
||||
key = args[0]
|
||||
page = kwargs.get(PAGE_KEY, 0)
|
||||
if method == b'findNode':
|
||||
result = self.node_rpc.find_node(sender_contact, key)
|
||||
else:
|
||||
assert method == b'findValue'
|
||||
result = self.node_rpc.find_value(sender_contact, key, page)
|
||||
|
||||
self.send_response(
|
||||
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
|
||||
)
|
||||
|
||||
def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
|
||||
# This is an RPC method request
|
||||
self.peer_manager.report_last_requested(address[0], address[1])
|
||||
try:
|
||||
peer = self.routing_table.get_peer(request_datagram.node_id)
|
||||
except IndexError:
|
||||
peer = make_kademlia_peer(request_datagram.node_id, address[0], address[1])
|
||||
try:
|
||||
self._handle_rpc(peer, request_datagram)
|
||||
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
|
||||
# will be added to our routing table if successful
|
||||
is_good = self.peer_manager.peer_is_good(peer)
|
||||
if is_good is None:
|
||||
self.ping_queue.enqueue_maybe_ping(peer)
|
||||
# only add a requesting contact to the routing table if it has replied to one of our requests
|
||||
elif is_good is True:
|
||||
self.add_peer(peer)
|
||||
except ValueError as err:
|
||||
log.debug("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
str(err))
|
||||
self.send_error(
|
||||
peer,
|
||||
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
|
||||
str(err).encode())
|
||||
)
|
||||
except Exception as err:
|
||||
log.warning("error raised handling %s request from %s:%i - %s(%s)",
|
||||
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
|
||||
str(err))
|
||||
self.send_error(
|
||||
peer,
|
||||
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
|
||||
str(err).encode())
|
||||
)
|
||||
|
||||
def handle_response_datagram(self, address: typing.Tuple[str, int], response_datagram: ResponseDatagram):
|
||||
# Find the message that triggered this response
|
||||
if response_datagram.rpc_id in self.sent_messages:
|
||||
peer, future, _ = self.sent_messages[response_datagram.rpc_id]
|
||||
if peer.address != address[0]:
|
||||
future.set_exception(
|
||||
RemoteException(f"response from {address[0]}, expected {peer.address}")
|
||||
)
|
||||
return
|
||||
|
||||
# We got a result from the RPC
|
||||
if peer.node_id == self.node_id:
|
||||
future.set_exception(RemoteException("node has our node id"))
|
||||
return
|
||||
elif response_datagram.node_id == self.node_id:
|
||||
future.set_exception(RemoteException("incoming message is from our node id"))
|
||||
return
|
||||
peer = make_kademlia_peer(response_datagram.node_id, address[0], address[1])
|
||||
self.peer_manager.report_last_replied(address[0], address[1])
|
||||
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
|
||||
if not future.cancelled():
|
||||
future.set_result(response_datagram)
|
||||
self.add_peer(peer)
|
||||
else:
|
||||
log.warning("%s:%i replied, but after we cancelled the request attempt",
|
||||
peer.address, peer.udp_port)
|
||||
else:
|
||||
# If the original message isn't found, it must have timed out
|
||||
# TODO: we should probably do something with this...
|
||||
pass
|
||||
|
||||
def handle_error_datagram(self, address, error_datagram: ErrorDatagram):
|
||||
# The RPC request raised a remote exception; raise it locally
|
||||
remote_exception = RemoteException(f"{error_datagram.exception_type}({error_datagram.response})")
|
||||
if error_datagram.rpc_id in self.sent_messages:
|
||||
peer, future, request = self.sent_messages.pop(error_datagram.rpc_id)
|
||||
if (peer.address, peer.udp_port) != address:
|
||||
future.set_exception(
|
||||
RemoteException(
|
||||
f"response from {address[0]}:{address[1]}, "
|
||||
f"expected {peer.address}:{peer.udp_port}"
|
||||
)
|
||||
)
|
||||
return
|
||||
error_msg = f"" \
|
||||
f"Error sending '{request.method}' to {peer.address}:{peer.udp_port}\n" \
|
||||
f"Args: {request.args}\n" \
|
||||
f"Raised: {str(remote_exception)}"
|
||||
if 'Invalid token' in error_msg:
|
||||
log.debug(error_msg)
|
||||
elif error_datagram.response not in OLD_PROTOCOL_ERRORS:
|
||||
log.warning(error_msg)
|
||||
else:
|
||||
log.debug(
|
||||
"known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
peer.address, peer.udp_port, OLD_PROTOCOL_ERRORS[error_datagram.response]
|
||||
)
|
||||
future.set_exception(remote_exception)
|
||||
return
|
||||
else:
|
||||
if error_datagram.response not in OLD_PROTOCOL_ERRORS:
|
||||
msg = f"Received error from {address[0]}:{address[1]}, but it isn't in response to a " \
|
||||
f"pending request: {str(remote_exception)}"
|
||||
log.warning(msg)
|
||||
else:
|
||||
log.debug(
|
||||
"known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
|
||||
address[0], address[1], OLD_PROTOCOL_ERRORS[error_datagram.response]
|
||||
)
|
||||
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError):
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
return
|
||||
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.handle_request_datagram(address, message)
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
self.handle_error_datagram(address, message)
|
||||
else:
|
||||
assert isinstance(message, ResponseDatagram), "sanity"
|
||||
self.handle_response_datagram(address, message)
|
||||
|
||||
async def send_request(self, peer: 'KademliaPeer', request: RequestDatagram) -> ResponseDatagram:
|
||||
self._send(peer, request)
|
||||
response_fut = self.sent_messages[request.rpc_id][1]
|
||||
try:
|
||||
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
|
||||
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
|
||||
return response
|
||||
except asyncio.CancelledError:
|
||||
if not response_fut.done():
|
||||
response_fut.cancel()
|
||||
raise
|
||||
except (asyncio.TimeoutError, RemoteException):
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
if self.peer_manager.peer_is_good(peer) is False:
|
||||
self.remove_peer(peer)
|
||||
raise
|
||||
|
||||
def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
|
||||
self._send(peer, response)
|
||||
|
||||
def send_error(self, peer: 'KademliaPeer', error: ErrorDatagram):
|
||||
self._send(peer, error)
|
||||
|
||||
def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]):
|
||||
if not self.transport or self.transport.is_closing():
|
||||
raise TransportNotConnected()
|
||||
|
||||
data = message.bencode()
|
||||
if len(data) > constants.MSG_SIZE_LIMIT:
|
||||
log.warning("cannot send datagram larger than %i bytes (packet is %i bytes)",
|
||||
constants.MSG_SIZE_LIMIT, len(data))
|
||||
log.debug("Packet is too large to send: %s", binascii.hexlify(data[:3500]).decode())
|
||||
raise ValueError(
|
||||
f"cannot send datagram larger than {constants.MSG_SIZE_LIMIT} bytes (packet is {len(data)} bytes)"
|
||||
)
|
||||
if isinstance(message, (RequestDatagram, ResponseDatagram)):
|
||||
assert message.node_id == self.node_id, message
|
||||
if isinstance(message, RequestDatagram):
|
||||
assert self.node_id != peer.node_id
|
||||
|
||||
def pop_from_sent_messages(_):
|
||||
if message.rpc_id in self.sent_messages:
|
||||
self.sent_messages.pop(message.rpc_id)
|
||||
|
||||
if isinstance(message, RequestDatagram):
|
||||
response_fut = self.loop.create_future()
|
||||
response_fut.add_done_callback(pop_from_sent_messages)
|
||||
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
|
||||
try:
|
||||
self.transport.sendto(data, (peer.address, peer.udp_port))
|
||||
except OSError as err:
|
||||
# TODO: handle ENETUNREACH
|
||||
if err.errno == socket.EWOULDBLOCK:
|
||||
# i'm scared this may swallow important errors, but i get a million of these
|
||||
# on Linux and it doesn't seem to affect anything -grin
|
||||
log.warning("Can't send data to dht: EWOULDBLOCK")
|
||||
else:
|
||||
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
|
||||
len(data), peer.address, peer.udp_port, str(err), err.errno)
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.sent_messages[message.rpc_id][1].set_exception(err)
|
||||
else:
|
||||
raise err
|
||||
if isinstance(message, RequestDatagram):
|
||||
self.peer_manager.report_last_sent(peer.address, peer.udp_port)
|
||||
elif isinstance(message, ErrorDatagram):
|
||||
self.peer_manager.report_failure(peer.address, peer.udp_port)
|
||||
|
||||
def change_token(self):
|
||||
self.old_token_secret = self.token_secret
|
||||
self.token_secret = constants.generate_id()
|
||||
|
||||
def make_token(self, compact_ip):
|
||||
return constants.digest(self.token_secret + compact_ip)
|
||||
|
||||
def verify_token(self, token, compact_ip):
|
||||
h = constants.HASH_CLASS()
|
||||
h.update(self.token_secret + compact_ip)
|
||||
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
|
||||
h = constants.HASH_CLASS()
|
||||
h.update(self.old_token_secret + compact_ip)
|
||||
if not token == h.digest():
|
||||
return False
|
||||
return True
|
||||
|
||||
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer', # pylint: disable=too-many-return-statements
|
||||
retry: bool = True) -> typing.Tuple[bytes, bool]:
|
||||
async def __store():
|
||||
res = await self.get_rpc_peer(peer).store(hash_value)
|
||||
if res != b"OK":
|
||||
raise ValueError(res)
|
||||
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, True
|
||||
|
||||
try:
|
||||
return await __store()
|
||||
except asyncio.TimeoutError:
|
||||
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
|
||||
return peer.node_id, False
|
||||
except ValueError as err:
|
||||
log.error("Unexpected response: %s", err)
|
||||
return peer.node_id, False
|
||||
except RemoteException as err:
|
||||
if 'findValue() takes exactly 2 arguments (5 given)' in str(err):
|
||||
log.debug("peer %s:%i is running an incompatible version of lbrynet", peer.address, peer.udp_port)
|
||||
return peer.node_id, False
|
||||
if 'Invalid token' not in str(err):
|
||||
log.warning("Unexpected error while storing blob_hash: %s", err)
|
||||
return peer.node_id, False
|
||||
self.peer_manager.clear_token(peer.node_id)
|
||||
if not retry:
|
||||
return peer.node_id, False
|
||||
return await self.store_to_peer(hash_value, peer, retry=False)
|
||||
@@ -1,319 +0,0 @@
|
||||
import asyncio
|
||||
import random
|
||||
import logging
|
||||
import typing
|
||||
import itertools
|
||||
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.protocol.distance import Distance
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.dht.peer import KademliaPeer, PeerManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KBucket:
|
||||
""" Description - later
|
||||
"""
|
||||
|
||||
def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int, node_id: bytes):
|
||||
"""
|
||||
@param range_min: The lower boundary for the range in the n-bit ID
|
||||
space covered by this k-bucket
|
||||
@param range_max: The upper boundary for the range in the ID space
|
||||
covered by this k-bucket
|
||||
"""
|
||||
self._peer_manager = peer_manager
|
||||
self.last_accessed = 0
|
||||
self.range_min = range_min
|
||||
self.range_max = range_max
|
||||
self.peers: typing.List['KademliaPeer'] = []
|
||||
self._node_id = node_id
|
||||
self._distance_to_self = Distance(node_id)
|
||||
|
||||
def add_peer(self, peer: 'KademliaPeer') -> bool:
|
||||
""" Add contact to _contact list in the right order. This will move the
|
||||
contact to the end of the k-bucket if it is already present.
|
||||
|
||||
@raise kademlia.kbucket.BucketFull: Raised when the bucket is full and
|
||||
the contact isn't in the bucket
|
||||
already
|
||||
|
||||
@param peer: The contact to add
|
||||
@type peer: dht.contact._Contact
|
||||
"""
|
||||
if peer in self.peers:
|
||||
# Move the existing contact to the end of the list
|
||||
# - using the new contact to allow add-on data
|
||||
# (e.g. optimization-specific stuff) to pe updated as well
|
||||
self.peers.remove(peer)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
for i in range(len(self.peers)):
|
||||
local_peer = self.peers[i]
|
||||
if local_peer.node_id == peer.node_id:
|
||||
self.peers.remove(local_peer)
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
if len(self.peers) < constants.K:
|
||||
self.peers.append(peer)
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
# raise BucketFull("No space in bucket to insert contact")
|
||||
|
||||
def get_peer(self, node_id: bytes) -> 'KademliaPeer':
|
||||
for peer in self.peers:
|
||||
if peer.node_id == node_id:
|
||||
return peer
|
||||
raise IndexError(node_id)
|
||||
|
||||
def get_peers(self, count=-1, exclude_contact=None, sort_distance_to=None) -> typing.List['KademliaPeer']:
|
||||
""" Returns a list containing up to the first count number of contacts
|
||||
|
||||
@param count: The amount of contacts to return (if 0 or less, return
|
||||
all contacts)
|
||||
@type count: int
|
||||
@param exclude_contact: A node node_id to exclude; if this contact is in
|
||||
the list of returned values, it will be
|
||||
discarded before returning. If a C{str} is
|
||||
passed as this argument, it must be the
|
||||
contact's ID.
|
||||
@type exclude_contact: str
|
||||
|
||||
@param sort_distance_to: Sort distance to the node_id, defaulting to the parent node node_id. If False don't
|
||||
sort the contacts
|
||||
|
||||
@raise IndexError: If the number of requested contacts is too large
|
||||
|
||||
@return: Return up to the first count number of contacts in a list
|
||||
If no contacts are present an empty is returned
|
||||
@rtype: list
|
||||
"""
|
||||
peers = [peer for peer in self.peers if peer.node_id != exclude_contact]
|
||||
|
||||
# Return all contacts in bucket
|
||||
if count <= 0:
|
||||
count = len(peers)
|
||||
|
||||
# Get current contact number
|
||||
current_len = len(peers)
|
||||
|
||||
# If count greater than k - return only k contacts
|
||||
if count > constants.K:
|
||||
count = constants.K
|
||||
|
||||
if not current_len:
|
||||
return peers
|
||||
|
||||
if sort_distance_to is False:
|
||||
pass
|
||||
else:
|
||||
sort_distance_to = sort_distance_to or self._node_id
|
||||
peers.sort(key=lambda c: Distance(sort_distance_to)(c.node_id))
|
||||
|
||||
return peers[:min(current_len, count)]
|
||||
|
||||
def get_bad_or_unknown_peers(self) -> typing.List['KademliaPeer']:
|
||||
peer = self.get_peers(sort_distance_to=False)
|
||||
return [
|
||||
peer for peer in peer
|
||||
if self._peer_manager.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port) is not True
|
||||
]
|
||||
|
||||
def remove_peer(self, peer: 'KademliaPeer') -> None:
|
||||
self.peers.remove(peer)
|
||||
|
||||
def key_in_range(self, key: bytes) -> bool:
|
||||
""" Tests whether the specified key (i.e. node ID) is in the range
|
||||
of the n-bit ID space covered by this k-bucket (in otherwords, it
|
||||
returns whether or not the specified key should be placed in this
|
||||
k-bucket)
|
||||
|
||||
@param key: The key to test
|
||||
@type key: str or int
|
||||
|
||||
@return: C{True} if the key is in this k-bucket's range, or C{False}
|
||||
if not.
|
||||
@rtype: bool
|
||||
"""
|
||||
return self.range_min <= self._distance_to_self(key) < self.range_max
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.peers)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.peers
|
||||
|
||||
|
||||
class TreeRoutingTable:
|
||||
""" This class implements a routing table used by a Node class.
|
||||
|
||||
The Kademlia routing table is a binary tree whose leaves are k-buckets,
|
||||
where each k-bucket contains nodes with some common prefix of their IDs.
|
||||
This prefix is the k-bucket's position in the binary tree; it therefore
|
||||
covers some range of ID values, and together all of the k-buckets cover
|
||||
the entire n-bit ID (or key) space (with no overlap).
|
||||
|
||||
@note: In this implementation, nodes in the tree (the k-buckets) are
|
||||
added dynamically, as needed; this technique is described in the 13-page
|
||||
version of the Kademlia paper, in section 2.4. It does, however, use the
|
||||
ping RPC-based k-bucket eviction algorithm described in section 2.2 of
|
||||
that paper.
|
||||
"""
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
|
||||
split_buckets_under_index: int = constants.SPLIT_BUCKETS_UNDER_INDEX):
|
||||
self._loop = loop
|
||||
self._peer_manager = peer_manager
|
||||
self._parent_node_id = parent_node_id
|
||||
self._split_buckets_under_index = split_buckets_under_index
|
||||
self.buckets: typing.List[KBucket] = [
|
||||
KBucket(
|
||||
self._peer_manager, range_min=0, range_max=2 ** constants.HASH_BITS, node_id=self._parent_node_id
|
||||
)
|
||||
]
|
||||
|
||||
def get_peers(self) -> typing.List['KademliaPeer']:
|
||||
return list(itertools.chain.from_iterable(map(lambda bucket: bucket.peers, self.buckets)))
|
||||
|
||||
def should_split(self, bucket_index: int, to_add: bytes) -> bool:
|
||||
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
|
||||
if bucket_index < self._split_buckets_under_index:
|
||||
return True
|
||||
contacts = self.get_peers()
|
||||
distance = Distance(self._parent_node_id)
|
||||
contacts.sort(key=lambda c: distance(c.node_id))
|
||||
kth_contact = contacts[-1] if len(contacts) < constants.K else contacts[constants.K - 1]
|
||||
return distance(to_add) < distance(kth_contact.node_id)
|
||||
|
||||
def find_close_peers(self, key: bytes, count: typing.Optional[int] = None,
|
||||
sender_node_id: typing.Optional[bytes] = None) -> typing.List['KademliaPeer']:
|
||||
exclude = [self._parent_node_id]
|
||||
if sender_node_id:
|
||||
exclude.append(sender_node_id)
|
||||
count = count or constants.K
|
||||
distance = Distance(key)
|
||||
contacts = self.get_peers()
|
||||
contacts = [c for c in contacts if c.node_id not in exclude]
|
||||
if contacts:
|
||||
contacts.sort(key=lambda c: distance(c.node_id))
|
||||
return contacts[:min(count, len(contacts))]
|
||||
return []
|
||||
|
||||
def get_peer(self, contact_id: bytes) -> 'KademliaPeer':
|
||||
"""
|
||||
@raise IndexError: No contact with the specified contact ID is known
|
||||
by this node
|
||||
"""
|
||||
return self.buckets[self.kbucket_index(contact_id)].get_peer(contact_id)
|
||||
|
||||
def get_refresh_list(self, start_index: int = 0, force: bool = False) -> typing.List[bytes]:
|
||||
bucket_index = start_index
|
||||
refresh_ids = []
|
||||
now = int(self._loop.time())
|
||||
for bucket in self.buckets[start_index:]:
|
||||
if force or now - bucket.last_accessed >= constants.REFRESH_INTERVAL:
|
||||
to_search = self.midpoint_id_in_bucket_range(bucket_index)
|
||||
refresh_ids.append(to_search)
|
||||
bucket_index += 1
|
||||
return refresh_ids
|
||||
|
||||
def remove_peer(self, peer: 'KademliaPeer') -> None:
|
||||
if not peer.node_id:
|
||||
return
|
||||
bucket_index = self.kbucket_index(peer.node_id)
|
||||
try:
|
||||
self.buckets[bucket_index].remove_peer(peer)
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
def touch_kbucket(self, key: bytes) -> None:
|
||||
self.touch_kbucket_by_index(self.kbucket_index(key))
|
||||
|
||||
def touch_kbucket_by_index(self, bucket_index: int):
|
||||
self.buckets[bucket_index].last_accessed = int(self._loop.time())
|
||||
|
||||
def kbucket_index(self, key: bytes) -> int:
|
||||
i = 0
|
||||
for bucket in self.buckets:
|
||||
if bucket.key_in_range(key):
|
||||
return i
|
||||
else:
|
||||
i += 1
|
||||
return i
|
||||
|
||||
def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
|
||||
return Distance(
|
||||
self._parent_node_id
|
||||
)(random_id.to_bytes(constants.HASH_LENGTH, 'big')).to_bytes(constants.HASH_LENGTH, 'big')
|
||||
|
||||
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
|
||||
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
|
||||
return Distance(self._parent_node_id)(
|
||||
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.HASH_LENGTH, 'big')
|
||||
).to_bytes(constants.HASH_LENGTH, 'big')
|
||||
|
||||
def split_bucket(self, old_bucket_index: int) -> None:
|
||||
""" Splits the specified k-bucket into two new buckets which together
|
||||
cover the same range in the key/ID space
|
||||
|
||||
@param old_bucket_index: The index of k-bucket to split (in this table's
|
||||
list of k-buckets)
|
||||
@type old_bucket_index: int
|
||||
"""
|
||||
# Resize the range of the current (old) k-bucket
|
||||
old_bucket = self.buckets[old_bucket_index]
|
||||
split_point = old_bucket.range_max - (old_bucket.range_max - old_bucket.range_min) // 2
|
||||
# Create a new k-bucket to cover the range split off from the old bucket
|
||||
new_bucket = KBucket(self._peer_manager, split_point, old_bucket.range_max, self._parent_node_id)
|
||||
old_bucket.range_max = split_point
|
||||
# Now, add the new bucket into the routing table tree
|
||||
self.buckets.insert(old_bucket_index + 1, new_bucket)
|
||||
# Finally, copy all nodes that belong to the new k-bucket into it...
|
||||
for contact in old_bucket.peers:
|
||||
if new_bucket.key_in_range(contact.node_id):
|
||||
new_bucket.add_peer(contact)
|
||||
# ...and remove them from the old bucket
|
||||
for contact in new_bucket.peers:
|
||||
old_bucket.remove_peer(contact)
|
||||
|
||||
def join_buckets(self):
|
||||
if len(self.buckets) == 1:
|
||||
return
|
||||
to_pop = [i for i, bucket in enumerate(self.buckets) if len(bucket) == 0]
|
||||
if not to_pop:
|
||||
return
|
||||
log.info("join buckets %i", len(to_pop))
|
||||
bucket_index_to_pop = to_pop[0]
|
||||
assert len(self.buckets[bucket_index_to_pop]) == 0
|
||||
can_go_lower = bucket_index_to_pop - 1 >= 0
|
||||
can_go_higher = bucket_index_to_pop + 1 < len(self.buckets)
|
||||
assert can_go_higher or can_go_lower
|
||||
bucket = self.buckets[bucket_index_to_pop]
|
||||
if can_go_lower and can_go_higher:
|
||||
midpoint = ((bucket.range_max - bucket.range_min) // 2) + bucket.range_min
|
||||
self.buckets[bucket_index_to_pop - 1].range_max = midpoint - 1
|
||||
self.buckets[bucket_index_to_pop + 1].range_min = midpoint
|
||||
elif can_go_lower:
|
||||
self.buckets[bucket_index_to_pop - 1].range_max = bucket.range_max
|
||||
elif can_go_higher:
|
||||
self.buckets[bucket_index_to_pop + 1].range_min = bucket.range_min
|
||||
self.buckets.remove(bucket)
|
||||
return self.join_buckets()
|
||||
|
||||
def contact_in_routing_table(self, address_tuple: typing.Tuple[str, int]) -> bool:
|
||||
for bucket in self.buckets:
|
||||
for contact in bucket.get_peers(sort_distance_to=False):
|
||||
if address_tuple[0] == contact.address and address_tuple[1] == contact.udp_port:
|
||||
return True
|
||||
return False
|
||||
|
||||
def buckets_with_contacts(self) -> int:
|
||||
count = 0
|
||||
for bucket in self.buckets:
|
||||
if len(bucket) > 0:
|
||||
count += 1
|
||||
return count
|
||||
@@ -1,75 +0,0 @@
|
||||
import typing
|
||||
from lbry.dht.error import DecodeError
|
||||
|
||||
|
||||
def _bencode(data: typing.Union[int, bytes, bytearray, str, list, tuple, dict]) -> bytes:
|
||||
if isinstance(data, int):
|
||||
return b'i%de' % data
|
||||
elif isinstance(data, (bytes, bytearray)):
|
||||
return b'%d:%s' % (len(data), data)
|
||||
elif isinstance(data, str):
|
||||
return b'%d:%s' % (len(data), data.encode())
|
||||
elif isinstance(data, (list, tuple)):
|
||||
encoded_list_items = b''
|
||||
for item in data:
|
||||
encoded_list_items += _bencode(item)
|
||||
return b'l%se' % encoded_list_items
|
||||
elif isinstance(data, dict):
|
||||
encoded_dict_items = b''
|
||||
keys = data.keys()
|
||||
for key in sorted(keys):
|
||||
encoded_dict_items += _bencode(key)
|
||||
encoded_dict_items += _bencode(data[key])
|
||||
return b'd%se' % encoded_dict_items
|
||||
else:
|
||||
raise TypeError(f"Cannot bencode {type(data)}")
|
||||
|
||||
|
||||
def _bdecode(data: bytes, start_index: int = 0) -> typing.Tuple[typing.Union[int, bytes, list, tuple, dict], int]:
|
||||
if data[start_index] == ord('i'):
|
||||
end_pos = data[start_index:].find(b'e') + start_index
|
||||
return int(data[start_index + 1:end_pos]), end_pos + 1
|
||||
elif data[start_index] == ord('l'):
|
||||
start_index += 1
|
||||
decoded_list = []
|
||||
while data[start_index] != ord('e'):
|
||||
list_data, start_index = _bdecode(data, start_index)
|
||||
decoded_list.append(list_data)
|
||||
return decoded_list, start_index + 1
|
||||
elif data[start_index] == ord('d'):
|
||||
start_index += 1
|
||||
decoded_dict = {}
|
||||
while data[start_index] != ord('e'):
|
||||
key, start_index = _bdecode(data, start_index)
|
||||
value, start_index = _bdecode(data, start_index)
|
||||
decoded_dict[key] = value
|
||||
return decoded_dict, start_index
|
||||
else:
|
||||
split_pos = data[start_index:].find(b':') + start_index
|
||||
try:
|
||||
length = int(data[start_index:split_pos])
|
||||
except (ValueError, TypeError) as err:
|
||||
raise DecodeError(err)
|
||||
start_index = split_pos + 1
|
||||
end_pos = start_index + length
|
||||
return data[start_index:end_pos], end_pos
|
||||
|
||||
|
||||
def bencode(data: typing.Dict) -> bytes:
|
||||
if not isinstance(data, dict):
|
||||
raise TypeError()
|
||||
return _bencode(data)
|
||||
|
||||
|
||||
def bdecode(data: bytes, allow_non_dict_return: typing.Optional[bool] = False) -> typing.Dict:
|
||||
assert isinstance(data, bytes), DecodeError(f"invalid data type: {str(type(data))}")
|
||||
|
||||
if len(data) == 0:
|
||||
raise DecodeError('Cannot decode empty string')
|
||||
try:
|
||||
result = _bdecode(data)[0]
|
||||
if not allow_non_dict_return and not isinstance(result, dict):
|
||||
raise ValueError(f'expected dict, got {type(result)}')
|
||||
return result
|
||||
except (ValueError, TypeError) as err:
|
||||
raise DecodeError(err)
|
||||
Loaded 100 of 471 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user