Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5dc2dadb03 | ||
|
|
1af58c83f8 | ||
|
|
a10ae70773 |
No files matched your search
@@ -1,67 +0,0 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [ master ]
|
||||
schedule:
|
||||
- cron: '21 0 * * 4'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [ 'python' ]
|
||||
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
|
||||
# Learn more:
|
||||
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v1
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v1
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 https://git.io/JvXDl
|
||||
|
||||
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
||||
# and modify them (or add more) to build your code if your project
|
||||
# uses a compiled language
|
||||
|
||||
#- run: |
|
||||
# make bootstrap
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v1
|
||||
@@ -1,94 +0,0 @@
|
||||
name: ci
|
||||
on: pull_request
|
||||
jobs:
|
||||
|
||||
lint:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- run: make lint
|
||||
|
||||
tests-unit:
|
||||
name: "tests / unit"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- run: make install tools
|
||||
- env:
|
||||
HOME: /tmp
|
||||
run: make test-unit-coverage
|
||||
|
||||
tests-integration:
|
||||
name: "tests / integration"
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
test:
|
||||
- datanetwork
|
||||
- blockchain
|
||||
- other
|
||||
steps:
|
||||
- name: Configure sysctl limits
|
||||
run: |
|
||||
sudo swapoff -a
|
||||
sudo sysctl -w vm.swappiness=1
|
||||
sudo sysctl -w fs.file-max=262144
|
||||
sudo sysctl -w vm.max_map_count=262144
|
||||
|
||||
- name: Runs Elasticsearch
|
||||
uses: elastic/elastic-github-actions/elasticsearch@master
|
||||
with:
|
||||
stack-version: 7.6.0
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- if: matrix.test == 'other'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends ffmpeg
|
||||
- run: pip install tox-travis
|
||||
- run: tox -e ${{ matrix.test }}
|
||||
|
||||
build:
|
||||
needs: ["lint", "tests-unit", "tests-integration"]
|
||||
name: "build"
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.7'
|
||||
- name: Setup
|
||||
run: |
|
||||
pip install pyinstaller
|
||||
pip install -e .
|
||||
# https://stackoverflow.com/a/61693590
|
||||
# https://github.com/pypa/setuptools/issues/1963
|
||||
pip install --upgrade 'setuptools<45.0.0'
|
||||
- if: startsWith(matrix.os, 'windows') == false
|
||||
name: Build & Run (Unix)
|
||||
run: |
|
||||
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
|
||||
chmod +x dist/lbrynet
|
||||
dist/lbrynet --version
|
||||
- if: startsWith(matrix.os, 'windows')
|
||||
name: Build & Run (Windows)
|
||||
run: |
|
||||
pip install pywin32
|
||||
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
|
||||
dist/lbrynet.exe --version
|
||||
@@ -1,44 +0,0 @@
|
||||
# This workflow integrates a collection of open source static analysis tools
|
||||
# with GitHub code scanning. For documentation, or to provide feedback, visit
|
||||
# https://github.com/github/ossar-action
|
||||
name: OSSAR
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [ master ]
|
||||
schedule:
|
||||
- cron: '28 17 * * 1'
|
||||
|
||||
jobs:
|
||||
OSSAR-Scan:
|
||||
# OSSAR runs on windows-latest.
|
||||
# ubuntu-latest and macos-latest support coming soon
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
# Ensure a compatible version of dotnet is installed.
|
||||
# The [Microsoft Security Code Analysis CLI](https://aka.ms/mscadocs) is built with dotnet v3.1.201.
|
||||
# A version greater than or equal to v3.1.201 of dotnet must be installed on the agent in order to run this action.
|
||||
# GitHub hosted runners already have a compatible version of dotnet installed and this step may be skipped.
|
||||
# For self-hosted runners, ensure dotnet version 3.1.201 or later is installed by including this action:
|
||||
# - name: Install .NET
|
||||
# uses: actions/setup-dotnet@v1
|
||||
# with:
|
||||
# dotnet-version: '3.1.x'
|
||||
|
||||
# Run open source static analysis tools
|
||||
- name: Run OSSAR
|
||||
uses: github/ossar-action@v1
|
||||
id: ossar
|
||||
|
||||
# Upload results to the Security tab
|
||||
- name: Upload OSSAR results
|
||||
uses: github/codeql-action/upload-sarif@v1
|
||||
with:
|
||||
sarif_file: ${{ steps.ossar.outputs.sarifFile }}
|
||||
@@ -5,18 +5,12 @@
|
||||
/.tox
|
||||
/.coverage*
|
||||
/lbry-venv
|
||||
/venv
|
||||
/lbry/blockchain
|
||||
|
||||
lbry.egg-info
|
||||
__pycache__
|
||||
_trial_temp/
|
||||
trending*.log
|
||||
|
||||
/tests/integration/blockchain/files
|
||||
/tests/.coverage.*
|
||||
|
||||
/lbry/wallet/bin
|
||||
|
||||
/.vscode
|
||||
/.gitignore
|
||||
+7
-20
@@ -1,11 +1,5 @@
|
||||
default:
|
||||
image: python:3.7
|
||||
services:
|
||||
- name: docker.elastic.co/elasticsearch/elasticsearch:7.11.2
|
||||
alias: elasticsearch
|
||||
command: [ "bin/elasticsearch", "-Expack.security.enabled=false", "-Ediscovery.type=single-node" ]
|
||||
variables:
|
||||
ES_JAVA_OPTS: "-Xms512m -Xmx512m"
|
||||
|
||||
|
||||
#cache:
|
||||
@@ -47,28 +41,25 @@ test:datanetwork-integration:
|
||||
stage: test
|
||||
script:
|
||||
- pip install tox-travis
|
||||
- ELASTIC_HOST=elasticsearch tox -e datanetwork --recreate
|
||||
- tox -e datanetwork
|
||||
|
||||
test:blockchain-integration:
|
||||
stage: test
|
||||
script:
|
||||
- curl "http://elasticsearch:9200/_cat/health"
|
||||
- pip install tox-travis
|
||||
- ELASTIC_HOST=elasticsearch tox -e blockchain
|
||||
- tox -e blockchain
|
||||
|
||||
test:other-integration:
|
||||
stage: test
|
||||
script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends ffmpeg
|
||||
- pip install tox-travis
|
||||
- ELASTIC_HOST=elasticsearch tox -e other
|
||||
- tox -e other
|
||||
|
||||
test:json-api:
|
||||
stage: test
|
||||
script:
|
||||
- make install tools
|
||||
- ELASTIC_HOST=elasticsearch HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
- HOME=/tmp coverage run -p --source=lbry scripts/generate_json_api.py
|
||||
|
||||
|
||||
|
||||
@@ -91,17 +82,15 @@ test:json-api:
|
||||
|
||||
build:linux:
|
||||
extends: .build
|
||||
image: ubuntu:16.04
|
||||
image: ubuntu:18.04 # cant use python3.7 image - binary won't run on ubuntu
|
||||
variables:
|
||||
OS: linux
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- apt-get install -y --no-install-recommends python3.7-dev python3-setuptools python3-wheel
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
- pip install lbry-libtorrent
|
||||
|
||||
build:mac:
|
||||
extends: .build
|
||||
@@ -128,11 +117,9 @@ build:windows:
|
||||
GIT_DEPTH: 5
|
||||
before_script:
|
||||
- ./docker/install_choco.ps1
|
||||
- choco install -y --x64 python --version=3.7.9
|
||||
- choco install -y 7zip checksum
|
||||
- choco install -y --x86 python3 7zip checksum
|
||||
- python --version # | findstr /B "Python 3\.7\." # dont upgrade python on every run. just make sure we're on the right Python
|
||||
- pip --version # | findstr /E '\(python 3\.7\)'
|
||||
- python -c "import sys;print(f'{str(64 if sys.maxsize > 2**32 else 32)} bit python\n{sys.platform}')"
|
||||
- pip install virtualenv pywin32
|
||||
- virtualenv venv
|
||||
- venv/Scripts/activate.ps1
|
||||
|
||||
+1
-13
@@ -92,23 +92,11 @@ To install on Windows:
|
||||
```
|
||||
|
||||
## Run the tests
|
||||
### Elasticsearch
|
||||
|
||||
For running integration tests, Elasticsearch is required to be available at localhost:9200/
|
||||
|
||||
The easiest way to start it is using docker with:
|
||||
```bash
|
||||
make elastic-docker
|
||||
```
|
||||
Alternative installation methods are available [at Elasticsearch website](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
|
||||
|
||||
|
||||
|
||||
To run the unit and integration tests from the repo directory:
|
||||
To run the unit tests from the repo directory:
|
||||
|
||||
```
|
||||
python -m unittest discover tests.unit
|
||||
python -m unittest discover tests.integration
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015-2020 LBRY Inc
|
||||
Copyright (c) 2015-2019 LBRY Inc
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
.PHONY: install tools lint test test-unit test-unit-coverage test-integration idea
|
||||
.PHONY: install tools lint test idea
|
||||
|
||||
install:
|
||||
pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl
|
||||
CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \
|
||||
--global-option=fetch \
|
||||
--global-option=--version --global-option=3.30.1 --global-option=--all \
|
||||
@@ -9,27 +8,16 @@ install:
|
||||
pip install -e .
|
||||
|
||||
tools:
|
||||
pip install mypy==0.701 pylint==2.4.4
|
||||
pip install mypy==0.701
|
||||
pip install coverage astroid pylint
|
||||
|
||||
lint:
|
||||
pylint --rcfile=setup.cfg lbry
|
||||
#mypy --ignore-missing-imports lbry
|
||||
|
||||
test: test-unit test-integration
|
||||
|
||||
test-unit:
|
||||
python -m unittest discover tests.unit
|
||||
|
||||
test-unit-coverage:
|
||||
coverage run -p --source=lbry -m unittest discover -vv tests.unit
|
||||
|
||||
test-integration:
|
||||
test:
|
||||
tox
|
||||
|
||||
idea:
|
||||
mkdir -p .idea
|
||||
cp -r scripts/idea/* .idea
|
||||
|
||||
elastic-docker:
|
||||
docker run -d -v lbryhub:/usr/share/elasticsearch/data -p 9200:9200 -p 9300:9300 -e"ES_JAVA_OPTS=-Xms512m -Xmx512m" -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1
|
||||
@@ -41,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our PGP key is here](https://lbry.com/faq/pgp-key) if you need it.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
While we are not at v1.0 yet, only the latest release will be supported.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
See https://lbry.com/faq/security
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM debian:10-slim
|
||||
FROM ubuntu:19.10
|
||||
|
||||
ARG user=lbry
|
||||
ARG db_dir=/database
|
||||
@@ -13,9 +13,7 @@ RUN apt-get update && \
|
||||
wget \
|
||||
tar unzip \
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libleveldb-dev \
|
||||
python3.7 \
|
||||
python3 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
python3-wheel \
|
||||
|
||||
@@ -1,45 +1,34 @@
|
||||
version: "3"
|
||||
|
||||
volumes:
|
||||
lbrycrd:
|
||||
wallet_server:
|
||||
es01:
|
||||
|
||||
services:
|
||||
lbrycrd:
|
||||
image: lbry/lbrycrd:${LBRYCRD_TAG:-latest-release}
|
||||
restart: always
|
||||
ports: # accessible from host
|
||||
- "9246:9246" # rpc port
|
||||
expose: # internal to docker network. also this doesn't do anything. its for documentation only.
|
||||
- "9245" # node-to-node comms port
|
||||
volumes:
|
||||
- "lbrycrd:/data/.lbrycrd"
|
||||
environment:
|
||||
- RUN_MODE=default
|
||||
- SNAPSHOT_URL=${LBRYCRD_SNAPSHOT_URL-https://lbry.com/snapshot/blockchain}
|
||||
- RPC_ALLOW_IP=0.0.0.0/0
|
||||
wallet_server:
|
||||
depends_on:
|
||||
- es01
|
||||
image: lbry/wallet-server:${WALLET_SERVER_TAG:-latest-release}
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
restart: always
|
||||
network_mode: host
|
||||
ports:
|
||||
- "50001:50001" # rpc port
|
||||
- "2112:2112" # uncomment to enable prometheus
|
||||
- "50005:50005" # websocket port
|
||||
- "2112:2112" # prometheus
|
||||
volumes:
|
||||
- "wallet_server:/database"
|
||||
environment:
|
||||
- DAEMON_URL=http://lbry:lbry@127.0.0.1:9245
|
||||
- TCP_PORT=50001
|
||||
- PROMETHEUS_PORT=2112
|
||||
- QUERY_TIMEOUT_MS=3000 # how long search queries allowed to run before cancelling, in milliseconds
|
||||
- TRENDING_ALGORITHMS=variable_decay
|
||||
- MAX_SEND=10000000000000 # deprecated. leave it high until its removed
|
||||
- MAX_SUBS=1000000000000 # deprecated. leave it high until its removed
|
||||
- FILTERING_CHANNEL_IDS=770bd7ecba84fd2f7607fb15aedd2b172c2e153f 95e5db68a3101df19763f3a5182e4b12ba393ee8
|
||||
- BLOCKING_CHANNEL_IDS=dd687b357950f6f271999971f43c785e8067c3a9 06871aa438032244202840ec59a469b303257cad b4a2528f436eca1bf3bf3e10ff3f98c57bd6c4c6 e4e230b131082f6b10c8f7994bbb83f29e8e6fb9
|
||||
es01:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
|
||||
container_name: es01
|
||||
environment:
|
||||
- node.name=es01
|
||||
- discovery.type=single-node
|
||||
- indices.query.bool.max_clause_count=4096
|
||||
- bootstrap.memory_lock=true
|
||||
- "ES_JAVA_OPTS=-Xms8g -Xmx8g" # no more than 32, remember to disable swap
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
volumes:
|
||||
- es01:/usr/share/elasticsearch/data
|
||||
ports:
|
||||
- 127.0.0.1:9200:9200
|
||||
- SNAPSHOT_URL=${WALLET_SERVER_SNAPSHOT_URL-https://lbry.com/snapshot/wallet}
|
||||
- DAEMON_URL=http://lbry:lbry@lbrycrd:9245
|
||||
@@ -20,6 +20,4 @@ if [[ -n "$SNAPSHOT_URL" ]] && [[ ! -f /database/claims.db ]]; then
|
||||
rm "$filename"
|
||||
fi
|
||||
|
||||
/home/lbry/.local/bin/lbry-hub-elastic-sync /database/claims.db
|
||||
echo 'starting server'
|
||||
/home/lbry/.local/bin/lbry-hub "$@"
|
||||
/home/lbry/.local/bin/torba-server "$@"
|
||||
+206
-1353
File diff suppressed because it is too large.
Load diff
+1
-1
@@ -1,2 +1,2 @@
|
||||
__version__ = "0.99.0"
|
||||
__version__ = "0.54.1"
|
||||
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
|
||||
+11
-25
@@ -110,7 +110,7 @@ class AbstractBlob:
|
||||
if reader in self.readers:
|
||||
self.readers.remove(reader)
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes) -> asyncio.Task:
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_length(self, length) -> None:
|
||||
@@ -167,7 +167,7 @@ class AbstractBlob:
|
||||
with self.reader_context() as handle:
|
||||
try:
|
||||
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
|
||||
except (ConnectionError, BrokenPipeError, RuntimeError, OSError, AttributeError):
|
||||
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError, AttributeError):
|
||||
return -1
|
||||
|
||||
def decrypt(self, key: bytes, iv: bytes) -> bytes:
|
||||
@@ -198,17 +198,11 @@ class AbstractBlob:
|
||||
def save_verified_blob(self, verified_bytes: bytes):
|
||||
if self.verified.is_set():
|
||||
return
|
||||
|
||||
def update_events(_):
|
||||
self.verified.set()
|
||||
self.writing.clear()
|
||||
|
||||
if self.is_writeable():
|
||||
self.writing.set()
|
||||
task = self._write_blob(verified_bytes)
|
||||
task.add_done_callback(update_events)
|
||||
self._write_blob(verified_bytes)
|
||||
self.verified.set()
|
||||
if self.blob_completed_callback:
|
||||
task.add_done_callback(lambda _: self.blob_completed_callback(self))
|
||||
self.blob_completed_callback(self)
|
||||
|
||||
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
|
||||
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
|
||||
@@ -267,11 +261,9 @@ class BlobBuffer(AbstractBlob):
|
||||
self.verified.clear()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
async def write():
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
return self.loop.create_task(write())
|
||||
if self._verified_bytes:
|
||||
raise OSError("already have bytes for blob")
|
||||
self._verified_bytes = BytesIO(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
if self._verified_bytes:
|
||||
@@ -327,19 +319,13 @@ class BlobFile(AbstractBlob):
|
||||
handle.close()
|
||||
|
||||
def _write_blob(self, blob_bytes: bytes):
|
||||
def _write_blob():
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
async def write_blob():
|
||||
await self.loop.run_in_executor(None, _write_blob)
|
||||
|
||||
return self.loop.create_task(write_blob())
|
||||
with open(self.file_path, 'wb') as f:
|
||||
f.write(blob_bytes)
|
||||
|
||||
def delete(self):
|
||||
super().delete()
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
return super().delete()
|
||||
|
||||
@classmethod
|
||||
async def create_from_unencrypted(
|
||||
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.connection_manager import ConnectionManager
|
||||
@@ -32,7 +32,7 @@ class BlobManager:
|
||||
else self._node_data_store.completed_blobs
|
||||
self.blobs: typing.Dict[str, AbstractBlob] = {}
|
||||
self.config = config
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics(
|
||||
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
|
||||
self.config.blob_lru_cache_size)
|
||||
self.connection_manager = ConnectionManager(loop)
|
||||
|
||||
|
||||
@@ -152,8 +152,6 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
|
||||
log.debug(msg)
|
||||
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
|
||||
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
|
||||
# wait for the io to finish
|
||||
await self.blob.verified.wait()
|
||||
log.info("%s at %fMB/s", msg,
|
||||
round((float(self._blob_bytes_received) /
|
||||
float(time.perf_counter() - start_time)) / 1000000.0, 2))
|
||||
|
||||
@@ -105,21 +105,18 @@ class BlobServerProtocol(asyncio.Protocol):
|
||||
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, sent)
|
||||
log.info("sent %s (%i bytes) to %s:%i", blob_hash, sent, peer_address, peer_port)
|
||||
else:
|
||||
self.close()
|
||||
log.debug("stopped sending %s to %s:%i", blob_hash, peer_address, peer_port)
|
||||
return
|
||||
except (OSError, ValueError, asyncio.TimeoutError) as err:
|
||||
except (OSError, asyncio.TimeoutError) as err:
|
||||
if isinstance(err, asyncio.TimeoutError):
|
||||
log.debug("timed out sending blob %s to %s", blob_hash, peer_address)
|
||||
else:
|
||||
log.warning("could not read blob %s to send %s:%i", blob_hash, peer_address, peer_port)
|
||||
self.close()
|
||||
return
|
||||
finally:
|
||||
self.transfer_finished.set()
|
||||
else:
|
||||
log.info("don't have %s to send %s:%i", blob.blob_hash[:8], peer_address, peer_port)
|
||||
if responses and not self.transport.is_closing():
|
||||
if responses:
|
||||
self.send_response(responses)
|
||||
|
||||
def data_received(self, data):
|
||||
|
||||
+26
-68
@@ -5,8 +5,9 @@ import typing
|
||||
import logging
|
||||
from argparse import ArgumentParser
|
||||
from contextlib import contextmanager
|
||||
from appdirs import user_data_dir, user_config_dir
|
||||
|
||||
import yaml
|
||||
from appdirs import user_data_dir, user_config_dir
|
||||
from lbry.error import InvalidCurrencyError
|
||||
from lbry.dht import constants
|
||||
from lbry.wallet.coinselection import STRATEGIES
|
||||
@@ -276,23 +277,14 @@ class Strings(ListSetting):
|
||||
class EnvironmentAccess:
|
||||
PREFIX = 'LBRY_'
|
||||
|
||||
def __init__(self, config: 'BaseConfig', environ: dict):
|
||||
self.configuration = config
|
||||
self.data = {}
|
||||
if environ:
|
||||
self.load(environ)
|
||||
|
||||
def load(self, environ):
|
||||
for setting in self.configuration.get_settings():
|
||||
value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET)
|
||||
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
|
||||
self.data[setting.name] = setting.deserialize(value)
|
||||
def __init__(self, environ: dict):
|
||||
self.environ = environ
|
||||
|
||||
def __contains__(self, item: str):
|
||||
return item in self.data
|
||||
return f'{self.PREFIX}{item.upper()}' in self.environ
|
||||
|
||||
def __getitem__(self, item: str):
|
||||
return self.data[item]
|
||||
return self.environ[f'{self.PREFIX}{item.upper()}']
|
||||
|
||||
|
||||
class ArgumentAccess:
|
||||
@@ -333,7 +325,7 @@ class ConfigFileAccess:
|
||||
cls = type(self.configuration)
|
||||
with open(self.path, 'r') as config_file:
|
||||
raw = config_file.read()
|
||||
serialized = yaml.safe_load(raw) or {}
|
||||
serialized = yaml.load(raw) or {}
|
||||
for key, value in serialized.items():
|
||||
attr = getattr(cls, key, None)
|
||||
if attr is None:
|
||||
@@ -451,7 +443,7 @@ class BaseConfig:
|
||||
self.arguments = ArgumentAccess(self, args)
|
||||
|
||||
def set_environment(self, environ=None):
|
||||
self.environment = EnvironmentAccess(self, environ or os.environ)
|
||||
self.environment = EnvironmentAccess(environ or os.environ)
|
||||
|
||||
def set_persisted(self, config_file_path=None):
|
||||
if config_file_path is None:
|
||||
@@ -470,27 +462,7 @@ class BaseConfig:
|
||||
self.persisted.save()
|
||||
|
||||
|
||||
class TranscodeConfig(BaseConfig):
|
||||
|
||||
ffmpeg_path = String('A list of places to check for ffmpeg and ffprobe. '
|
||||
f'$data_dir/ffmpeg/bin and $PATH are checked afterward. Separator: {os.pathsep}',
|
||||
'', previous_names=['ffmpeg_folder'])
|
||||
video_encoder = String('FFmpeg codec and parameters for the video encoding. '
|
||||
'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental',
|
||||
'libx264 -crf 24 -preset faster -pix_fmt yuv420p')
|
||||
video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 5_000_000)
|
||||
video_scaler = String('FFmpeg scaling parameters for reducing bitrate. '
|
||||
'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M',
|
||||
r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" '
|
||||
r'-maxrate 5500K -bufsize 5000K')
|
||||
audio_encoder = String('FFmpeg codec and parameters for the audio encoding. '
|
||||
'Example: libopus -b:a 128k',
|
||||
'aac -b:a 160k')
|
||||
volume_filter = String('FFmpeg filter for audio normalization. Exmple: -af loudnorm', '')
|
||||
volume_analysis_time = Integer('Maximum seconds into the file that we examine audio volume (0 to disable).', 240)
|
||||
|
||||
|
||||
class CLIConfig(TranscodeConfig):
|
||||
class CLIConfig(BaseConfig):
|
||||
|
||||
api = String('Host name and port for lbrynet daemon API.', 'localhost:5279', metavar='HOST:PORT')
|
||||
|
||||
@@ -573,7 +545,6 @@ class Config(CLIConfig):
|
||||
"Don't download streams with fees exceeding this amount. When set to "
|
||||
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
|
||||
)
|
||||
max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0")
|
||||
|
||||
# reflector settings
|
||||
reflect_streams = Toggle(
|
||||
@@ -585,24 +556,19 @@ class Config(CLIConfig):
|
||||
)
|
||||
|
||||
# servers
|
||||
reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [
|
||||
reflector_servers = Servers("Reflector re-hosting servers", [
|
||||
('reflector.lbry.com', 5566)
|
||||
])
|
||||
|
||||
fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [
|
||||
('cdn.reflector.lbry.com', 5567)
|
||||
])
|
||||
|
||||
lbryum_servers = Servers("SPV wallet servers", [
|
||||
('spv11.lbry.com', 50001),
|
||||
('spv12.lbry.com', 50001),
|
||||
('spv13.lbry.com', 50001),
|
||||
('spv14.lbry.com', 50001),
|
||||
('spv15.lbry.com', 50001),
|
||||
('spv16.lbry.com', 50001),
|
||||
('spv17.lbry.com', 50001),
|
||||
('spv18.lbry.com', 50001),
|
||||
('spv19.lbry.com', 50001),
|
||||
('spv1.lbry.com', 50001),
|
||||
('spv2.lbry.com', 50001),
|
||||
('spv3.lbry.com', 50001),
|
||||
('spv4.lbry.com', 50001),
|
||||
('spv5.lbry.com', 50001),
|
||||
('spv6.lbry.com', 50001),
|
||||
('spv7.lbry.com', 50001),
|
||||
('spv8.lbry.com', 50001),
|
||||
('spv9.lbry.com', 50001),
|
||||
])
|
||||
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
|
||||
('lbrynet1.lbry.com', 4444), # US EAST
|
||||
@@ -611,22 +577,21 @@ class Config(CLIConfig):
|
||||
('lbrynet4.lbry.com', 4444) # ASIA
|
||||
])
|
||||
|
||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api/v2")
|
||||
comment_server = String("Comment server API URL", "https://comments.lbry.com/api")
|
||||
|
||||
# blockchain
|
||||
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
|
||||
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
|
||||
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
|
||||
|
||||
# daemon
|
||||
save_files = Toggle("Save downloaded files when calling `get` by default", True)
|
||||
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
|
||||
share_usage_data = Toggle(
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", False,
|
||||
"Whether to share usage stats and diagnostic info with LBRY.", True,
|
||||
previous_names=['upload_log', 'upload_log', 'share_debug_info']
|
||||
)
|
||||
track_bandwidth = Toggle("Track bandwidth usage", True)
|
||||
allowed_origin = String(
|
||||
"Allowed `Origin` header value for API request (sent by browser), use * to allow "
|
||||
"all hosts; default is to only allow API requests with no `Origin` value.", "")
|
||||
|
||||
# media server
|
||||
streaming_server = String('Host name and port to serve streaming media over range requests',
|
||||
@@ -638,12 +603,6 @@ class Config(CLIConfig):
|
||||
"Strategy to use when selecting UTXOs for a transaction",
|
||||
STRATEGIES, "standard")
|
||||
|
||||
transaction_cache_size = Integer("Transaction cache size", 2 ** 17)
|
||||
save_resolved_claims = Toggle(
|
||||
"Save content claims to the database when they are resolved to keep file_list up to date, "
|
||||
"only disable this if file_x commands are not needed", True
|
||||
)
|
||||
|
||||
@property
|
||||
def streaming_host(self):
|
||||
return self.streaming_server.split(':')[0]
|
||||
@@ -708,10 +667,9 @@ def get_darwin_directories() -> typing.Tuple[str, str, str]:
|
||||
def get_linux_directories() -> typing.Tuple[str, str, str]:
|
||||
try:
|
||||
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read())
|
||||
if down_dir:
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1))
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
|
||||
down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
|
||||
download_dir = re.sub('\"', '', down_dir)
|
||||
except OSError:
|
||||
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
|
||||
if not download_dir:
|
||||
|
||||
+22
-5
@@ -1,14 +1,14 @@
|
||||
import typing
|
||||
import asyncio
|
||||
import logging
|
||||
import ipaddress
|
||||
from binascii import hexlify
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4
|
||||
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -20,9 +20,26 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
|
||||
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
|
||||
|
||||
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
ALLOW_LOCALHOST = False
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False):
|
||||
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
|
||||
return _is_valid_public_ipv4(address, allow_localhost)
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
return not any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local,
|
||||
parsed_ip.is_loopback, parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private,
|
||||
parsed_ip.is_reserved,
|
||||
CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except ipaddress.AddressValueError:
|
||||
return False
|
||||
|
||||
|
||||
class PeerManager:
|
||||
@@ -155,9 +172,9 @@ class KademliaPeer:
|
||||
if self._node_id is not None:
|
||||
if not len(self._node_id) == constants.HASH_LENGTH:
|
||||
raise ValueError("invalid node_id: {}".format(hexlify(self._node_id).decode()))
|
||||
if self.udp_port is not None and not 1024 <= self.udp_port <= 65535:
|
||||
if self.udp_port is not None and not 1 <= self.udp_port <= 65535:
|
||||
raise ValueError("invalid udp port")
|
||||
if self.tcp_port is not None and not 1024 <= self.tcp_port <= 65535:
|
||||
if self.tcp_port is not None and not 1 <= self.tcp_port <= 65535:
|
||||
raise ValueError("invalid tcp port")
|
||||
if not is_valid_public_ipv4(self.address, self.allow_localhost):
|
||||
raise ValueError(f"invalid ip address: '{self.address}'")
|
||||
|
||||
@@ -10,7 +10,6 @@ from asyncio.protocols import DatagramProtocol
|
||||
from asyncio.transports import DatagramTransport
|
||||
|
||||
from lbry.dht import constants
|
||||
from lbry.dht.serialization.bencoding import DecodeError
|
||||
from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
|
||||
from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY
|
||||
from lbry.dht.error import RemoteException, TransportNotConnected
|
||||
@@ -555,7 +554,7 @@ class KademliaProtocol(DatagramProtocol):
|
||||
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ
|
||||
try:
|
||||
message = decode_datagram(datagram)
|
||||
except (ValueError, TypeError, DecodeError):
|
||||
except (ValueError, TypeError):
|
||||
self.peer_manager.report_failure(address[0], address[1])
|
||||
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
|
||||
return
|
||||
|
||||
@@ -144,7 +144,7 @@ class ErrorDatagram(KademliaDatagramBase):
|
||||
self.response = response.decode()
|
||||
|
||||
|
||||
def _decode_datagram(datagram: bytes):
|
||||
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
|
||||
msg_types = {
|
||||
REQUEST_TYPE: RequestDatagram,
|
||||
RESPONSE_TYPE: ResponseDatagram,
|
||||
@@ -152,29 +152,19 @@ def _decode_datagram(datagram: bytes):
|
||||
}
|
||||
|
||||
primitive: typing.Dict = bdecode(datagram)
|
||||
|
||||
converted = {
|
||||
str(k).encode() if not isinstance(k, bytes) else k: v for k, v in primitive.items()
|
||||
}
|
||||
|
||||
if converted[b'0'] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = converted[b'0'] # pylint: disable=unsubscriptable-object
|
||||
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
|
||||
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
|
||||
else:
|
||||
raise ValueError("invalid datagram type")
|
||||
datagram_class = msg_types[datagram_type]
|
||||
decoded = {
|
||||
k: converted[str(i).encode()] # pylint: disable=unsubscriptable-object
|
||||
k: primitive[i] # pylint: disable=unsubscriptable-object
|
||||
for i, k in enumerate(datagram_class.required_fields)
|
||||
if str(i).encode() in converted # pylint: disable=unsupported-membership-test
|
||||
if i in primitive # pylint: disable=unsupported-membership-test
|
||||
}
|
||||
for i, _ in enumerate(OPTIONAL_FIELDS):
|
||||
if str(i + OPTIONAL_ARG_OFFSET).encode() in converted:
|
||||
decoded[i + OPTIONAL_ARG_OFFSET] = converted[str(i + OPTIONAL_ARG_OFFSET).encode()]
|
||||
return decoded, datagram_class
|
||||
|
||||
|
||||
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
|
||||
decoded, datagram_class = _decode_datagram(datagram)
|
||||
if i + OPTIONAL_ARG_OFFSET in primitive:
|
||||
decoded[i + OPTIONAL_ARG_OFFSET] = primitive[i + OPTIONAL_ARG_OFFSET]
|
||||
return datagram_class(**decoded)
|
||||
|
||||
|
||||
|
||||
@@ -51,15 +51,11 @@ Code | Name | Message
|
||||
405 | ChannelKeyNotFound | Channel signing key not found.
|
||||
406 | ChannelKeyInvalid | Channel signing key is out of date. -- For example, channel was updated but you don't have the updated key.
|
||||
407 | DataDownload | Failed to download blob. *generic*
|
||||
410 | Resolve | Failed to resolve '{url}'.
|
||||
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
|
||||
420 | KeyFeeAboveMaxAllowed | {message}
|
||||
421 | InvalidPassword | Password is invalid.
|
||||
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
|
||||
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
|
||||
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
|
||||
408 | Resolve | Failed to resolve '{url}'.
|
||||
409 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
|
||||
410 | KeyFeeAboveMaxAllowed | {message}
|
||||
411 | InvalidPassword | Password is invalid.
|
||||
412 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
|
||||
**5xx** | Blob | **Blobs**
|
||||
500 | BlobNotFound | Blob not found.
|
||||
501 | BlobPermissionDenied | Permission denied to read blob.
|
||||
|
||||
+1
-60
@@ -1,4 +1,4 @@
|
||||
from .base import BaseError, claim_id
|
||||
from .base import BaseError
|
||||
|
||||
|
||||
class UserInputError(BaseError):
|
||||
@@ -16,22 +16,18 @@ class CommandError(UserInputError):
|
||||
class CommandDoesNotExistError(CommandError):
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' does not exist.")
|
||||
|
||||
|
||||
class CommandDeprecatedError(CommandError):
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' is deprecated.")
|
||||
|
||||
|
||||
class CommandInvalidArgumentError(CommandError):
|
||||
|
||||
def __init__(self, argument, command):
|
||||
self.argument = argument
|
||||
self.command = command
|
||||
super().__init__(f"Invalid argument '{argument}' to command '{command}'.")
|
||||
|
||||
|
||||
@@ -41,7 +37,6 @@ class CommandTemporarilyUnavailableError(CommandError):
|
||||
"""
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' is temporarily unavailable.")
|
||||
|
||||
|
||||
@@ -51,7 +46,6 @@ class CommandPermanentlyUnavailableError(CommandError):
|
||||
"""
|
||||
|
||||
def __init__(self, command):
|
||||
self.command = command
|
||||
super().__init__(f"Command '{command}' is permanently unavailable.")
|
||||
|
||||
|
||||
@@ -64,15 +58,12 @@ class InputValueError(UserInputError, ValueError):
|
||||
class GenericInputValueError(InputValueError):
|
||||
|
||||
def __init__(self, value, argument):
|
||||
self.value = value
|
||||
self.argument = argument
|
||||
super().__init__(f"The value '{value}' for argument '{argument}' is not valid.")
|
||||
|
||||
|
||||
class InputValueIsNoneError(InputValueError):
|
||||
|
||||
def __init__(self, argument):
|
||||
self.argument = argument
|
||||
super().__init__(f"None or null is not valid value for argument '{argument}'.")
|
||||
|
||||
|
||||
@@ -88,7 +79,6 @@ class ConfigWriteError(ConfigurationError):
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Cannot write configuration file '{path}'.")
|
||||
|
||||
|
||||
@@ -98,7 +88,6 @@ class ConfigReadError(ConfigurationError):
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Cannot find provided configuration file '{path}'.")
|
||||
|
||||
|
||||
@@ -108,21 +97,18 @@ class ConfigParseError(ConfigurationError):
|
||||
"""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Failed to parse the configuration file '{path}'.")
|
||||
|
||||
|
||||
class ConfigMissingError(ConfigurationError):
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Configuration file '{path}' is missing setting that has no default / fallback.")
|
||||
|
||||
|
||||
class ConfigInvalidError(ConfigurationError):
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
super().__init__(f"Configuration file '{path}' has setting with invalid value.")
|
||||
|
||||
|
||||
@@ -202,29 +188,18 @@ class DataDownloadError(WalletError):
|
||||
class ResolveError(WalletError):
|
||||
|
||||
def __init__(self, url):
|
||||
self.url = url
|
||||
super().__init__(f"Failed to resolve '{url}'.")
|
||||
|
||||
|
||||
class ResolveTimeoutError(WalletError):
|
||||
|
||||
def __init__(self, url):
|
||||
self.url = url
|
||||
super().__init__(f"Failed to resolve '{url}' within the timeout.")
|
||||
|
||||
|
||||
class ResolveCensoredError(WalletError):
|
||||
|
||||
def __init__(self, url, censor_hash):
|
||||
self.url = url
|
||||
self.censor_hash = censor_hash
|
||||
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.")
|
||||
|
||||
|
||||
class KeyFeeAboveMaxAllowedError(WalletError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
@@ -237,32 +212,9 @@ class InvalidPasswordError(WalletError):
|
||||
class IncompatibleWalletServerError(WalletError):
|
||||
|
||||
def __init__(self, server, port):
|
||||
self.server = server
|
||||
self.port = port
|
||||
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
|
||||
|
||||
|
||||
class ServerPaymentInvalidAddressError(WalletError):
|
||||
|
||||
def __init__(self, address):
|
||||
self.address = address
|
||||
super().__init__(f"Invalid address from wallet server: '{address}' - skipping payment round.")
|
||||
|
||||
|
||||
class ServerPaymentWalletLockedError(WalletError):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("Cannot spend funds with locked wallet, skipping payment round.")
|
||||
|
||||
|
||||
class ServerPaymentFeeAboveMaxAllowedError(WalletError):
|
||||
|
||||
def __init__(self, daily_fee, max_fee):
|
||||
self.daily_fee = daily_fee
|
||||
self.max_fee = max_fee
|
||||
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
|
||||
|
||||
|
||||
class BlobError(BaseError):
|
||||
"""
|
||||
**Blobs**
|
||||
@@ -320,35 +272,30 @@ class DownloadCancelledError(BlobError):
|
||||
class DownloadSDTimeoutError(BlobError):
|
||||
|
||||
def __init__(self, download):
|
||||
self.download = download
|
||||
super().__init__(f"Failed to download sd blob {download} within timeout.")
|
||||
|
||||
|
||||
class DownloadDataTimeoutError(BlobError):
|
||||
|
||||
def __init__(self, download):
|
||||
self.download = download
|
||||
super().__init__(f"Failed to download data blobs for sd hash {download} within timeout.")
|
||||
|
||||
|
||||
class InvalidStreamDescriptorError(BlobError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidDataError(BlobError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidBlobHashError(BlobError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
@@ -361,14 +308,12 @@ class ComponentError(BaseError):
|
||||
class ComponentStartConditionNotMetError(ComponentError):
|
||||
|
||||
def __init__(self, components):
|
||||
self.components = components
|
||||
super().__init__(f"Unresolved dependencies for: {components}")
|
||||
|
||||
|
||||
class ComponentsNotStartedError(ComponentError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
@@ -381,20 +326,16 @@ class CurrencyExchangeError(BaseError):
|
||||
class InvalidExchangeRateResponseError(CurrencyExchangeError):
|
||||
|
||||
def __init__(self, source, reason):
|
||||
self.source = source
|
||||
self.reason = reason
|
||||
super().__init__(f"Failed to get exchange rate from {source}: {reason}")
|
||||
|
||||
|
||||
class CurrencyConversionError(CurrencyExchangeError):
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
super().__init__(f"{message}")
|
||||
|
||||
|
||||
class InvalidCurrencyError(CurrencyExchangeError):
|
||||
|
||||
def __init__(self, currency):
|
||||
self.currency = currency
|
||||
super().__init__(f"Invalid currency: {currency} is not a supported currency.")
|
||||
@@ -1,9 +1,2 @@
|
||||
from binascii import hexlify
|
||||
|
||||
|
||||
def claim_id(claim_hash):
|
||||
return hexlify(claim_hash[::-1]).decode()
|
||||
|
||||
|
||||
class BaseError(Exception):
|
||||
pass
|
||||
+4
-17
@@ -13,12 +13,10 @@ class {name}({parents}):{doc}
|
||||
"""
|
||||
|
||||
INIT = """
|
||||
def __init__({args}):{fields}
|
||||
def __init__({args}):
|
||||
super().__init__({format}"{message}")
|
||||
"""
|
||||
|
||||
FUNCTIONS = ['claim_id']
|
||||
|
||||
|
||||
class ErrorClass:
|
||||
|
||||
@@ -52,20 +50,10 @@ class ErrorClass:
|
||||
|
||||
def get_arguments(self):
|
||||
args = ['self']
|
||||
for arg in re.findall('{([a-z0-1_()]+)}', self.message):
|
||||
for func in FUNCTIONS:
|
||||
if arg.startswith(f'{func}('):
|
||||
arg = arg[len(f'{func}('):-1]
|
||||
break
|
||||
for arg in re.findall('{([a-z0-1]+)}', self.message):
|
||||
args.append(arg)
|
||||
return args
|
||||
|
||||
@staticmethod
|
||||
def get_fields(args):
|
||||
if len(args) > 1:
|
||||
return f''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
|
||||
return ''
|
||||
|
||||
@staticmethod
|
||||
def get_doc_string(doc):
|
||||
if doc:
|
||||
@@ -81,8 +69,7 @@ class ErrorClass:
|
||||
args = self.get_arguments()
|
||||
if self.is_leaf:
|
||||
out.write((CLASS + INIT).format(
|
||||
name=self.class_name, parents=', '.join(parents),
|
||||
args=', '.join(args), fields=self.get_fields(args),
|
||||
name=self.class_name, parents=', '.join(parents), args=', '.join(args),
|
||||
message=self.message, doc=self.get_doc_string(self.comment), format='f' if len(args) > 1 else ''
|
||||
))
|
||||
else:
|
||||
@@ -115,7 +102,7 @@ def find_parent(stack, child):
|
||||
|
||||
|
||||
def generate(out):
|
||||
out.write(f"from .base import BaseError, {', '.join(FUNCTIONS)}\n")
|
||||
out.write('from .base import BaseError\n')
|
||||
stack = {}
|
||||
for error in get_errors():
|
||||
error.render(out, find_parent(stack, error))
|
||||
|
||||
@@ -14,6 +14,7 @@ from aiohttp.web import GracefulExit
|
||||
from docopt import docopt
|
||||
|
||||
from lbry import __version__ as lbrynet_version
|
||||
from lbry.extras.daemon.loggly_handler import get_loggly_handler
|
||||
from lbry.extras.daemon.daemon import Daemon
|
||||
from lbry.conf import Config, CLIConfig
|
||||
|
||||
@@ -254,6 +255,11 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
|
||||
else:
|
||||
logger.getChild('lbry').setLevel(logging.DEBUG)
|
||||
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
logger.getChild('lbry').addHandler(loggly_handler)
|
||||
|
||||
|
||||
def run_daemon(args: argparse.Namespace, conf: Config):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
@@ -66,7 +66,7 @@ def _download_properties(conf: Config, external_ip: str, resolve_duration: float
|
||||
"node_rpc_timeout": conf.node_rpc_timeout,
|
||||
"peer_connect_timeout": conf.peer_connect_timeout,
|
||||
"blob_download_timeout": conf.blob_download_timeout,
|
||||
"use_fixed_peers": len(conf.fixed_peers) > 0,
|
||||
"use_fixed_peers": len(conf.reflector_servers) > 0,
|
||||
"fixed_peer_delay": fixed_peer_delay,
|
||||
"added_fixed_peers": added_fixed_peers,
|
||||
"active_peer_count": active_peer_count,
|
||||
@@ -110,6 +110,7 @@ class AnalyticsManager:
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._enabled = conf.share_usage_data
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.context = _make_context(system_info.get_platform())
|
||||
self.installation_id = installation_id
|
||||
@@ -117,24 +118,20 @@ class AnalyticsManager:
|
||||
self.task: typing.Optional[asyncio.Task] = None
|
||||
self.external_ip: typing.Optional[str] = None
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self.conf.share_usage_data
|
||||
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
async def start(self):
|
||||
if self.task is None:
|
||||
if self._enabled and self.task is None:
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
self.task = asyncio.create_task(self.run())
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
if self.enabled:
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
await self._send_heartbeat()
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
@@ -157,7 +154,7 @@ class AnalyticsManager:
|
||||
|
||||
async def track(self, event: typing.Dict):
|
||||
"""Send a single tracking event"""
|
||||
if self.enabled:
|
||||
if self._enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post(event)
|
||||
|
||||
|
||||
@@ -22,54 +22,41 @@ def cid2hash(claim_id: str) -> bytes:
|
||||
return binascii.unhexlify(claim_id.encode())[::-1]
|
||||
|
||||
|
||||
def is_comment_signed_by_channel(comment: dict, channel: Output, sign_comment_id=False):
|
||||
def is_comment_signed_by_channel(comment: dict, channel: Output, abandon=False):
|
||||
if isinstance(channel, Output):
|
||||
try:
|
||||
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
|
||||
return verify(channel, signing_field.encode(), comment, cid2hash(comment['channel_id']))
|
||||
signing_field = comment['comment_id'] if abandon else comment['comment']
|
||||
pieces = [
|
||||
comment['signing_ts'].encode(),
|
||||
cid2hash(comment['channel_id']),
|
||||
signing_field.encode()
|
||||
]
|
||||
return Output.is_signature_valid(
|
||||
get_encoded_signature(comment['signature']),
|
||||
sha256(b''.join(pieces)),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
except KeyError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def verify(channel, data, signature, channel_hash=None):
|
||||
pieces = [
|
||||
signature['signing_ts'].encode(),
|
||||
channel_hash or channel.claim_hash,
|
||||
data
|
||||
]
|
||||
return Output.is_signature_valid(
|
||||
get_encoded_signature(signature['signature']),
|
||||
sha256(b''.join(pieces)),
|
||||
channel.claim.channel.public_key_bytes
|
||||
)
|
||||
|
||||
|
||||
def sign_comment(comment: dict, channel: Output, sign_comment_id=False):
|
||||
signing_field = comment['comment_id'] if sign_comment_id else comment['comment']
|
||||
comment.update(sign(channel, signing_field.encode()))
|
||||
|
||||
|
||||
def sign(channel, data):
|
||||
def sign_comment(comment: dict, channel: Output, abandon=False):
|
||||
timestamp = str(int(time.time()))
|
||||
pieces = [timestamp.encode(), channel.claim_hash, data]
|
||||
signing_field = comment['comment_id'] if abandon else comment['comment']
|
||||
pieces = [timestamp.encode(), channel.claim_hash, signing_field.encode()]
|
||||
digest = sha256(b''.join(pieces))
|
||||
signature = channel.private_key.sign_digest_deterministic(digest, hashfunc=hashlib.sha256)
|
||||
return {
|
||||
comment.update({
|
||||
'signature': binascii.hexlify(signature).decode(),
|
||||
'signing_ts': timestamp
|
||||
}
|
||||
|
||||
|
||||
def sign_reaction(reaction: dict, channel: Output):
|
||||
signing_field = reaction['channel_name']
|
||||
reaction.update(sign(channel, signing_field.encode()))
|
||||
})
|
||||
|
||||
|
||||
async def jsonrpc_post(url: str, method: str, params: dict = None, **kwargs) -> any:
|
||||
params = params or {}
|
||||
params.update(kwargs)
|
||||
json_body = {'jsonrpc': '2.0', 'id': 1, 'method': method, 'params': params}
|
||||
json_body = {'jsonrpc': '2.0', 'id': None, 'method': method, 'params': params}
|
||||
async with utils.aiohttp_request('POST', url, json=json_body) as response:
|
||||
try:
|
||||
result = await response.json()
|
||||
|
||||
@@ -158,14 +158,11 @@ class ComponentManager:
|
||||
for component in self.components
|
||||
}
|
||||
|
||||
def get_actual_component(self, component_name):
|
||||
def get_component(self, component_name):
|
||||
for component in self.components:
|
||||
if component.component_name == component_name:
|
||||
return component
|
||||
return component.component
|
||||
raise NameError(component_name)
|
||||
|
||||
def get_component(self, component_name):
|
||||
return self.get_actual_component(component_name).component
|
||||
|
||||
def has_component(self, component_name):
|
||||
return any(component for component in self.components if component_name == component.component_name)
|
||||
@@ -17,17 +17,10 @@ from lbry.dht.blob_announcer import BlobAnnouncer
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.blob_exchange.server import BlobServer
|
||||
from lbry.stream.stream_manager import StreamManager
|
||||
from lbry.file.file_manager import FileManager
|
||||
from lbry.extras.daemon.component import Component
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.torrent.torrent_manager import TorrentManager
|
||||
from lbry.wallet import WalletManager
|
||||
from lbry.wallet.usage_payment import WalletServerPayer
|
||||
try:
|
||||
from lbry.torrent.session import TorrentSession
|
||||
except ImportError:
|
||||
TorrentSession = None
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -36,14 +29,12 @@ log = logging.getLogger(__name__)
|
||||
DATABASE_COMPONENT = "database"
|
||||
BLOB_COMPONENT = "blob_manager"
|
||||
WALLET_COMPONENT = "wallet"
|
||||
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
|
||||
DHT_COMPONENT = "dht"
|
||||
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
|
||||
FILE_MANAGER_COMPONENT = "file_manager"
|
||||
STREAM_MANAGER_COMPONENT = "stream_manager"
|
||||
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
|
||||
UPNP_COMPONENT = "upnp"
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
|
||||
LIBTORRENT_COMPONENT = "libtorrent_component"
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
@@ -119,17 +110,15 @@ class WalletComponent(Component):
|
||||
async def get_status(self):
|
||||
if self.wallet_manager is None:
|
||||
return
|
||||
is_connected = self.wallet_manager.ledger.network.is_connected
|
||||
sessions = []
|
||||
session_pool = self.wallet_manager.ledger.network.session_pool
|
||||
sessions = session_pool.sessions
|
||||
connected = None
|
||||
if is_connected:
|
||||
addr, port = self.wallet_manager.ledger.network.client.server
|
||||
connected = f"{addr}:{port}"
|
||||
sessions.append(self.wallet_manager.ledger.network.client)
|
||||
|
||||
if self.wallet_manager.ledger.network.client:
|
||||
addr_and_port = self.wallet_manager.ledger.network.client.server_address_and_port
|
||||
if addr_and_port:
|
||||
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
|
||||
result = {
|
||||
'connected': connected,
|
||||
'connected_features': self.wallet_manager.ledger.network.server_features,
|
||||
'servers': [
|
||||
{
|
||||
'host': session.server[0],
|
||||
@@ -138,8 +127,8 @@ class WalletComponent(Component):
|
||||
'availability': session.available,
|
||||
} for session in sessions
|
||||
],
|
||||
'known_servers': len(self.wallet_manager.ledger.network.config['default_servers']),
|
||||
'available_servers': 1 if is_connected else 0
|
||||
'known_servers': len(sessions),
|
||||
'available_servers': len(list(session_pool.available_sessions))
|
||||
}
|
||||
|
||||
if self.wallet_manager.ledger.network.remote_height:
|
||||
@@ -151,7 +140,7 @@ class WalletComponent(Component):
|
||||
progress = min(max(math.ceil(float(download_height) / float(target_height) * 100), 0), 100)
|
||||
else:
|
||||
progress = 100
|
||||
best_hash = await self.wallet_manager.get_best_blockhash()
|
||||
best_hash = self.wallet_manager.get_best_blockhash()
|
||||
result.update({
|
||||
'headers_synchronization_progress': progress,
|
||||
'blocks': max(local_height, 0),
|
||||
@@ -171,34 +160,6 @@ class WalletComponent(Component):
|
||||
self.wallet_manager = None
|
||||
|
||||
|
||||
class WalletServerPaymentsComponent(Component):
|
||||
component_name = WALLET_SERVER_PAYMENTS_COMPONENT
|
||||
depends_on = [WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.usage_payment_service = WalletServerPayer(
|
||||
max_fee=self.conf.max_wallet_server_fee, analytics_manager=self.component_manager.analytics_manager,
|
||||
)
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[WalletServerPayer]:
|
||||
return self.usage_payment_service
|
||||
|
||||
async def start(self):
|
||||
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
await self.usage_payment_service.start(wallet_manager.ledger, wallet_manager.default_wallet)
|
||||
|
||||
async def stop(self):
|
||||
await self.usage_payment_service.stop()
|
||||
|
||||
async def get_status(self):
|
||||
return {
|
||||
'max_fee': self.usage_payment_service.max_fee,
|
||||
'running': self.usage_payment_service.running
|
||||
}
|
||||
|
||||
|
||||
class BlobComponent(Component):
|
||||
component_name = BLOB_COMPONENT
|
||||
depends_on = [DATABASE_COMPONENT]
|
||||
@@ -275,7 +236,7 @@ class DHTComponent(Component):
|
||||
external_ip = upnp_component.external_ip
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
if not external_ip:
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
external_ip = await utils.get_external_ip()
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
@@ -327,23 +288,23 @@ class HashAnnouncerComponent(Component):
|
||||
}
|
||||
|
||||
|
||||
class FileManagerComponent(Component):
|
||||
component_name = FILE_MANAGER_COMPONENT
|
||||
class StreamManagerComponent(Component):
|
||||
component_name = STREAM_MANAGER_COMPONENT
|
||||
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.file_manager: typing.Optional[FileManager] = None
|
||||
self.stream_manager: typing.Optional[StreamManager] = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[FileManager]:
|
||||
return self.file_manager
|
||||
def component(self) -> typing.Optional[StreamManager]:
|
||||
return self.stream_manager
|
||||
|
||||
async def get_status(self):
|
||||
if not self.file_manager:
|
||||
if not self.stream_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.file_manager.get_filtered()),
|
||||
'managed_files': len(self.stream_manager.streams),
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
@@ -352,55 +313,16 @@ class FileManagerComponent(Component):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
node = self.component_manager.get_component(DHT_COMPONENT) \
|
||||
if self.component_manager.has_component(DHT_COMPONENT) else None
|
||||
try:
|
||||
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
|
||||
except NameError:
|
||||
torrent = None
|
||||
log.info('Starting the file manager')
|
||||
loop = asyncio.get_event_loop()
|
||||
self.file_manager = FileManager(
|
||||
loop, self.conf, wallet, storage, self.component_manager.analytics_manager
|
||||
self.stream_manager = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
|
||||
)
|
||||
self.file_manager.source_managers['stream'] = StreamManager(
|
||||
loop, self.conf, blob_manager, wallet, storage, node,
|
||||
)
|
||||
if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
|
||||
self.file_manager.source_managers['torrent'] = TorrentManager(
|
||||
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
|
||||
)
|
||||
await self.file_manager.start()
|
||||
await self.stream_manager.start()
|
||||
log.info('Done setting up file manager')
|
||||
|
||||
async def stop(self):
|
||||
self.file_manager.stop()
|
||||
|
||||
|
||||
class TorrentComponent(Component):
|
||||
component_name = LIBTORRENT_COMPONENT
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.torrent_session = None
|
||||
|
||||
@property
|
||||
def component(self) -> typing.Optional[TorrentSession]:
|
||||
return self.torrent_session
|
||||
|
||||
async def get_status(self):
|
||||
if not self.torrent_session:
|
||||
return
|
||||
return {
|
||||
'running': True, # TODO: what to return here?
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
if TorrentSession:
|
||||
self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
|
||||
await self.torrent_session.bind() # TODO: specify host/port
|
||||
|
||||
async def stop(self):
|
||||
if self.torrent_session:
|
||||
await self.torrent_session.pause()
|
||||
self.stream_manager.stop()
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -476,7 +398,7 @@ class UPnPComponent(Component):
|
||||
pass
|
||||
if external_ip and not is_valid_public_ipv4(external_ip):
|
||||
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
|
||||
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
external_ip = await utils.get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
if external_ip:
|
||||
@@ -534,15 +456,13 @@ class UPnPComponent(Component):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
return
|
||||
success = False
|
||||
await self._maintain_redirects()
|
||||
if self.upnp:
|
||||
if not self.upnp_redirects and not all(
|
||||
x in self.component_manager.skip_components
|
||||
for x in (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)
|
||||
):
|
||||
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
|
||||
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
|
||||
log.error("failed to setup upnp")
|
||||
else:
|
||||
success = True
|
||||
@@ -551,9 +471,9 @@ class UPnPComponent(Component):
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
if not self.external_ip:
|
||||
self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
|
||||
self.external_ip = await utils.get_external_ip()
|
||||
if self.external_ip:
|
||||
log.info("detected external ip using %s fallback", probed_url)
|
||||
log.info("detected external ip using lbry.com fallback")
|
||||
if self.component_manager.analytics_manager:
|
||||
self.component_manager.loop.create_task(
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(
|
||||
|
||||
+315
-1133
File diff suppressed because it is too large.
Load diff
@@ -2,10 +2,9 @@ import json
|
||||
import time
|
||||
import asyncio
|
||||
import logging
|
||||
from statistics import median
|
||||
from decimal import Decimal
|
||||
from typing import Optional, Iterable, Type
|
||||
from aiohttp.client_exceptions import ContentTypeError, ClientConnectionError
|
||||
from aiohttp.client_exceptions import ContentTypeError
|
||||
from lbry.error import InvalidExchangeRateResponseError, CurrencyConversionError
|
||||
from lbry.utils import aiohttp_request
|
||||
from lbry.wallet.dewies import lbc_to_dewies
|
||||
@@ -59,12 +58,9 @@ class MarketFeed:
|
||||
raise NotImplementedError()
|
||||
|
||||
async def get_response(self):
|
||||
async with aiohttp_request(
|
||||
'get', self.url, params=self.params,
|
||||
timeout=self.request_timeout, headers={"User-Agent": "lbrynet"}
|
||||
) as response:
|
||||
async with aiohttp_request('get', self.url, params=self.params, timeout=self.request_timeout) as response:
|
||||
try:
|
||||
self._last_response = await response.json(content_type=None)
|
||||
self._last_response = await response.json()
|
||||
except ContentTypeError as e:
|
||||
self._last_response = {}
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.message)
|
||||
@@ -79,23 +75,18 @@ class MarketFeed:
|
||||
log.debug("Saving rate update %f for %s from %s", rate, self.market, self.name)
|
||||
self.rate = ExchangeRate(self.market, rate, int(time.time()))
|
||||
self.last_check = time.time()
|
||||
self.event.set()
|
||||
return self.rate
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Timed out fetching exchange rate from %s.", self.name)
|
||||
except json.JSONDecodeError as e:
|
||||
msg = e.doc if '<html>' not in e.doc else 'unexpected content type.'
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, msg)
|
||||
log.debug(e.doc)
|
||||
log.warning("Could not parse exchange rate response from %s: %s", self.name, e.doc)
|
||||
except InvalidExchangeRateResponseError as e:
|
||||
log.warning(str(e))
|
||||
except ClientConnectionError as e:
|
||||
log.warning("Error trying to connect to exchange rate %s: %s", self.name, str(e))
|
||||
except Exception as e:
|
||||
log.exception("Exchange rate error (%s from %s):", self.market, self.name)
|
||||
finally:
|
||||
self.event.set()
|
||||
|
||||
async def keep_updated(self):
|
||||
while True:
|
||||
@@ -113,92 +104,70 @@ class MarketFeed:
|
||||
self.event.clear()
|
||||
|
||||
|
||||
class BaseBittrexFeed(MarketFeed):
|
||||
class BittrexFeed(MarketFeed):
|
||||
name = "Bittrex"
|
||||
market = None
|
||||
url = None
|
||||
market = "BTCLBC"
|
||||
url = "https://bittrex.com/api/v1.1/public/getmarkethistory"
|
||||
params = {'market': 'BTC-LBC', 'count': 50}
|
||||
fee = 0.0025
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'lastTradeRate' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response['lastTradeRate'])
|
||||
|
||||
|
||||
class BittrexBTCFeed(BaseBittrexFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.bittrex.com/v3/markets/LBC-BTC/ticker"
|
||||
|
||||
|
||||
class BittrexUSDFeed(BaseBittrexFeed):
|
||||
market = "USDLBC"
|
||||
url = "https://api.bittrex.com/v3/markets/LBC-USD/ticker"
|
||||
|
||||
|
||||
class BaseCoinExFeed(MarketFeed):
|
||||
name = "CoinEx"
|
||||
market = None
|
||||
url = None
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response or \
|
||||
'ticker' not in json_response['data'] or \
|
||||
'last' not in json_response['data']['ticker']:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response['data']['ticker']['last'])
|
||||
|
||||
|
||||
class CoinExBTCFeed(BaseCoinExFeed):
|
||||
market = "BTCLBC"
|
||||
url = "https://api.coinex.com/v1/market/ticker?market=LBCBTC"
|
||||
|
||||
|
||||
class CoinExUSDFeed(BaseCoinExFeed):
|
||||
market = "USDLBC"
|
||||
url = "https://api.coinex.com/v1/market/ticker?market=LBCUSDT"
|
||||
|
||||
|
||||
class BaseHotbitFeed(MarketFeed):
|
||||
name = "hotbit"
|
||||
market = None
|
||||
url = "https://api.hotbit.io/api/v1/market.last"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'result' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response['result'])
|
||||
trades = json_response['result']
|
||||
if len(trades) == 0:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'trades not found')
|
||||
totals = sum([i['Total'] for i in trades])
|
||||
qtys = sum([i['Quantity'] for i in trades])
|
||||
if totals <= 0 or qtys <= 0:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'quantities were not positive')
|
||||
vwap = totals / qtys
|
||||
return float(1.0 / vwap)
|
||||
|
||||
|
||||
class HotbitBTCFeed(BaseHotbitFeed):
|
||||
class LBRYFeed(MarketFeed):
|
||||
name = "lbry.com"
|
||||
market = "BTCLBC"
|
||||
params = {"market": "LBC/BTC"}
|
||||
|
||||
|
||||
class HotbitUSDFeed(BaseHotbitFeed):
|
||||
market = "USDLBC"
|
||||
params = {"market": "LBC/USDT"}
|
||||
|
||||
|
||||
class UPbitBTCFeed(MarketFeed):
|
||||
name = "UPbit"
|
||||
market = "BTCLBC"
|
||||
url = "https://api.upbit.com/v1/ticker"
|
||||
params = {"markets": "BTC-LBC"}
|
||||
url = "https://api.lbry.com/lbc/exchange_rate"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if len(json_response) != 1 or 'trade_price' not in json_response[0]:
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / float(json_response[0]['trade_price'])
|
||||
return 1.0 / json_response['data']['lbc_btc']
|
||||
|
||||
|
||||
class LBRYBTCFeed(LBRYFeed):
|
||||
market = "USDBTC"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return 1.0 / json_response['data']['btc_usd']
|
||||
|
||||
|
||||
class CryptonatorFeed(MarketFeed):
|
||||
name = "cryptonator.com"
|
||||
market = "BTCLBC"
|
||||
url = "https://api.cryptonator.com/api/ticker/btc-lbc"
|
||||
|
||||
def get_rate_from_response(self, json_response):
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponseError(self.name, 'result not found')
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class CryptonatorBTCFeed(CryptonatorFeed):
|
||||
market = "USDBTC"
|
||||
url = "https://api.cryptonator.com/api/ticker/usd-btc"
|
||||
|
||||
|
||||
FEEDS: Iterable[Type[MarketFeed]] = (
|
||||
BittrexBTCFeed,
|
||||
BittrexUSDFeed,
|
||||
CoinExBTCFeed,
|
||||
CoinExUSDFeed,
|
||||
HotbitBTCFeed,
|
||||
HotbitUSDFeed,
|
||||
UPbitBTCFeed,
|
||||
LBRYFeed,
|
||||
LBRYBTCFeed,
|
||||
BittrexFeed,
|
||||
# CryptonatorFeed,
|
||||
# CryptonatorBTCFeed,
|
||||
)
|
||||
|
||||
|
||||
@@ -222,23 +191,20 @@ class ExchangeRateManager:
|
||||
source.stop()
|
||||
|
||||
def convert_currency(self, from_currency, to_currency, amount):
|
||||
log.debug(
|
||||
"Converting %f %s to %s, rates: %s",
|
||||
amount, from_currency, to_currency,
|
||||
[market.rate for market in self.market_feeds]
|
||||
)
|
||||
rates = [market.rate for market in self.market_feeds]
|
||||
log.debug("Converting %f %s to %s, rates: %s", amount, from_currency, to_currency, rates)
|
||||
if from_currency == to_currency:
|
||||
return round(amount, 8)
|
||||
|
||||
rates = []
|
||||
for market in self.market_feeds:
|
||||
if (market.has_rate and market.is_online and
|
||||
market.rate.currency_pair == (from_currency, to_currency)):
|
||||
rates.append(market.rate.spot)
|
||||
|
||||
if rates:
|
||||
return round(amount * Decimal(median(rates)), 8)
|
||||
|
||||
return round(amount * Decimal(market.rate.spot), 8)
|
||||
for market in self.market_feeds:
|
||||
if (market.has_rate and market.is_online and
|
||||
market.rate.currency_pair[0] == from_currency):
|
||||
return round(self.convert_currency(
|
||||
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot)), 8)
|
||||
raise CurrencyConversionError(
|
||||
f'Unable to convert {amount} from {from_currency} to {to_currency}')
|
||||
|
||||
|
||||
@@ -7,8 +7,6 @@ from json import JSONEncoder
|
||||
from google.protobuf.message import DecodeError
|
||||
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.schema.support import Support
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.wallet import Wallet, Ledger, Account, Transaction, Output
|
||||
from lbry.wallet.bip32 import PubKey
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
@@ -27,8 +25,6 @@ def encode_txo_doc():
|
||||
'address': "address of who can spend the txo",
|
||||
'confirmations': "number of confirmed blocks",
|
||||
'is_change': "payment to change address, only available when it can be determined",
|
||||
'is_received': "true if txo was sent from external account to this account",
|
||||
'is_spent': "true if txo is spent",
|
||||
'is_mine': "payment to one of your accounts, only available when it can be determined",
|
||||
'type': "one of 'claim', 'support' or 'purchase'",
|
||||
'name': "when type is 'claim' or 'support', this is the claim name",
|
||||
@@ -110,9 +106,7 @@ def encode_file_doc():
|
||||
'metadata': '(dict) None if claim is not found else the claim metadata',
|
||||
'channel_claim_id': '(str) None if claim is not found or not signed',
|
||||
'channel_name': '(str) None if claim is not found or not signed',
|
||||
'claim_name': '(str) None if claim is not found else the claim name',
|
||||
'reflector_progress': '(int) reflector upload progress, 0 to 100',
|
||||
'uploading_to_reflector': '(bool) set to True when currently uploading to reflector'
|
||||
'claim_name': '(str) None if claim is not found else the claim name'
|
||||
}
|
||||
|
||||
|
||||
@@ -128,7 +122,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
return self.encode_account(obj)
|
||||
if isinstance(obj, Wallet):
|
||||
return self.encode_wallet(obj)
|
||||
if isinstance(obj, (ManagedStream, TorrentSource)):
|
||||
if isinstance(obj, ManagedStream):
|
||||
return self.encode_file(obj)
|
||||
if isinstance(obj, Transaction):
|
||||
return self.encode_transaction(obj)
|
||||
@@ -136,8 +130,6 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
return self.encode_output(obj)
|
||||
if isinstance(obj, Claim):
|
||||
return self.encode_claim(obj)
|
||||
if isinstance(obj, Support):
|
||||
return obj.to_dict()
|
||||
if isinstance(obj, PubKey):
|
||||
return obj.extended_key_string()
|
||||
if isinstance(obj, datetime):
|
||||
@@ -172,22 +164,12 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'address': txo.get_address(self.ledger) if txo.has_address else None,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers.estimated_timestamp(tx_height)
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
|
||||
}
|
||||
if txo.is_spent is not None:
|
||||
output['is_spent'] = txo.is_spent
|
||||
if txo.is_my_output is not None:
|
||||
output['is_my_output'] = txo.is_my_output
|
||||
if txo.is_my_input is not None:
|
||||
output['is_my_input'] = txo.is_my_input
|
||||
if txo.sent_supports is not None:
|
||||
output['sent_supports'] = dewies_to_lbc(txo.sent_supports)
|
||||
if txo.sent_tips is not None:
|
||||
output['sent_tips'] = dewies_to_lbc(txo.sent_tips)
|
||||
if txo.received_tips is not None:
|
||||
output['received_tips'] = dewies_to_lbc(txo.received_tips)
|
||||
if txo.is_internal_transfer is not None:
|
||||
output['is_internal_transfer'] = txo.is_internal_transfer
|
||||
if txo.is_change is not None:
|
||||
output['is_change'] = txo.is_change
|
||||
if txo.is_my_account is not None:
|
||||
output['is_mine'] = txo.is_my_account
|
||||
|
||||
if txo.script.is_claim_name:
|
||||
output['type'] = 'claim'
|
||||
@@ -223,28 +205,25 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['claims'] = [self.encode_output(o) for o in txo.claims]
|
||||
if txo.reposted_claim is not None:
|
||||
output['reposted_claim'] = self.encode_output(txo.reposted_claim)
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim or txo.script.is_support_claim_data:
|
||||
try:
|
||||
output['value'] = txo.signable
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.signable.to_bytes())
|
||||
if txo.purchase_receipt is not None:
|
||||
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
if txo.script.is_claim_name or txo.script.is_update_claim:
|
||||
try:
|
||||
output['value'] = txo.claim
|
||||
output['value_type'] = txo.claim.claim_type
|
||||
if self.include_protobuf:
|
||||
output['protobuf'] = hexlify(txo.claim.to_bytes())
|
||||
if txo.purchase_receipt is not None:
|
||||
output['purchase_receipt'] = self.encode_output(txo.purchase_receipt)
|
||||
if txo.claim.is_channel:
|
||||
output['has_signing_key'] = txo.has_private_key
|
||||
elif txo.script.is_support_claim_data:
|
||||
output['value_type'] = 'emoji'
|
||||
if check_signature and txo.signable.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
else:
|
||||
output['signing_channel'] = {'channel_id': txo.signable.signing_channel_id}
|
||||
output['is_channel_signature_valid'] = False
|
||||
except DecodeError:
|
||||
pass
|
||||
if check_signature and txo.claim.is_signed:
|
||||
if txo.channel is not None:
|
||||
output['signing_channel'] = self.encode_output(txo.channel)
|
||||
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
|
||||
else:
|
||||
output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id}
|
||||
output['is_channel_signature_valid'] = False
|
||||
except DecodeError:
|
||||
pass
|
||||
return output
|
||||
|
||||
def encode_claim_meta(self, meta):
|
||||
@@ -253,7 +232,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
if isinstance(value, int):
|
||||
meta[key] = dewies_to_lbc(value)
|
||||
if 0 < meta.get('creation_height', 0) <= self.ledger.headers.height:
|
||||
meta['creation_timestamp'] = self.ledger.headers.estimated_timestamp(meta['creation_height'])
|
||||
meta['creation_timestamp'] = self.ledger.headers[meta['creation_height']]['timestamp']
|
||||
return meta
|
||||
|
||||
def encode_input(self, txi):
|
||||
@@ -280,32 +259,26 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output_exists = managed_stream.output_file_exists
|
||||
tx_height = managed_stream.stream_claim_info.height
|
||||
best_height = self.ledger.headers.height
|
||||
is_stream = hasattr(managed_stream, 'stream_hash')
|
||||
if is_stream:
|
||||
total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length()
|
||||
total_bytes = managed_stream.descriptor.upper_bound_decrypted_length()
|
||||
else:
|
||||
total_bytes_lower_bound = total_bytes = managed_stream.torrent_length
|
||||
result = {
|
||||
'streaming_url': None,
|
||||
return {
|
||||
'streaming_url': managed_stream.stream_url,
|
||||
'completed': managed_stream.completed,
|
||||
'file_name': None,
|
||||
'download_directory': None,
|
||||
'download_path': None,
|
||||
'file_name': managed_stream.file_name if output_exists else None,
|
||||
'download_directory': managed_stream.download_directory if output_exists else None,
|
||||
'download_path': managed_stream.full_path if output_exists else None,
|
||||
'points_paid': 0.0,
|
||||
'stopped': not managed_stream.running,
|
||||
'stream_hash': None,
|
||||
'stream_name': None,
|
||||
'suggested_file_name': None,
|
||||
'sd_hash': None,
|
||||
'mime_type': None,
|
||||
'key': None,
|
||||
'total_bytes_lower_bound': total_bytes_lower_bound,
|
||||
'total_bytes': total_bytes,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(),
|
||||
'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(),
|
||||
'written_bytes': managed_stream.written_bytes,
|
||||
'blobs_completed': None,
|
||||
'blobs_in_stream': None,
|
||||
'blobs_remaining': None,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'status': managed_stream.status,
|
||||
'claim_id': managed_stream.claim_id,
|
||||
'txid': managed_stream.txid,
|
||||
@@ -321,38 +294,8 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
'added_on': managed_stream.added_on,
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
|
||||
'timestamp': self.ledger.headers.estimated_timestamp(tx_height),
|
||||
'is_fully_reflected': False,
|
||||
'reflector_progress': False,
|
||||
'uploading_to_reflector': False
|
||||
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
|
||||
}
|
||||
if is_stream:
|
||||
result.update({
|
||||
'streaming_url': managed_stream.stream_url,
|
||||
'stream_hash': managed_stream.stream_hash,
|
||||
'stream_name': managed_stream.descriptor.stream_name,
|
||||
'suggested_file_name': managed_stream.descriptor.suggested_file_name,
|
||||
'sd_hash': managed_stream.descriptor.sd_hash,
|
||||
'mime_type': managed_stream.mime_type,
|
||||
'key': managed_stream.descriptor.key,
|
||||
'blobs_completed': managed_stream.blobs_completed,
|
||||
'blobs_in_stream': managed_stream.blobs_in_stream,
|
||||
'blobs_remaining': managed_stream.blobs_remaining,
|
||||
'is_fully_reflected': managed_stream.is_fully_reflected,
|
||||
'reflector_progress': managed_stream.reflector_progress,
|
||||
'uploading_to_reflector': managed_stream.uploading_to_reflector
|
||||
})
|
||||
else:
|
||||
result.update({
|
||||
'streaming_url': f'file://{managed_stream.full_path}',
|
||||
})
|
||||
if output_exists:
|
||||
result.update({
|
||||
'file_name': managed_stream.file_name,
|
||||
'download_directory': managed_stream.download_directory,
|
||||
'download_path': managed_stream.full_path,
|
||||
})
|
||||
return result
|
||||
|
||||
def encode_claim(self, claim):
|
||||
encoded = getattr(claim, claim.claim_type).to_dict()
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
|
||||
from aiohttp.client_exceptions import ClientError
|
||||
import aiohttp
|
||||
from lbry import utils, __version__
|
||||
|
||||
|
||||
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
class HTTPSLogglyHandler(logging.Handler):
|
||||
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
|
||||
super().__init__()
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.cookies = cookies or {}
|
||||
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
|
||||
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
|
||||
)
|
||||
self._loop = asyncio.get_event_loop()
|
||||
self._session = aiohttp.ClientSession()
|
||||
|
||||
@staticmethod
|
||||
def get_full_message(record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
async def _emit(self, record, retry=True):
|
||||
data = self.format(record).encode()
|
||||
try:
|
||||
async with self._session.post(self.url, data=data,
|
||||
cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except ClientError:
|
||||
if self._loop.is_running() and retry:
|
||||
await self._session.close()
|
||||
self._session = aiohttp.ClientSession()
|
||||
return await self._emit(record, retry=False)
|
||||
|
||||
def emit(self, record):
|
||||
try:
|
||||
asyncio.ensure_future(self._emit(record), loop=self._loop)
|
||||
except RuntimeError: # TODO: use a second loop
|
||||
print(f"\nfailed to send traceback to loggly, please file an issue with the following traceback:\n"
|
||||
f"{self.format(record)}")
|
||||
|
||||
def close(self):
|
||||
super().close()
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(self._session.close())
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
|
||||
def get_loggly_handler():
|
||||
handler = HTTPSLogglyHandler(LOGGLY_TOKEN)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
return handler
|
||||
@@ -1,31 +0,0 @@
|
||||
import logging
|
||||
from aiohttp import web
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ensure_request_allowed(request, conf):
|
||||
if is_request_allowed(request, conf):
|
||||
return
|
||||
if conf.allowed_origin:
|
||||
log.warning(
|
||||
"API requests with Origin '%s' are not allowed, "
|
||||
"configuration 'allowed_origin' limits requests to: '%s'",
|
||||
request.headers.get('Origin'), conf.allowed_origin
|
||||
)
|
||||
else:
|
||||
log.warning(
|
||||
"API requests with Origin '%s' are not allowed, "
|
||||
"update configuration 'allowed_origin' to enable this origin.",
|
||||
request.headers.get('Origin')
|
||||
)
|
||||
raise web.HTTPForbidden()
|
||||
|
||||
|
||||
def is_request_allowed(request, conf) -> bool:
|
||||
origin = request.headers.get('Origin')
|
||||
return (
|
||||
origin is None or
|
||||
origin == conf.allowed_origin or
|
||||
conf.allowed_origin == '*'
|
||||
)
|
||||
@@ -9,7 +9,7 @@ from typing import Optional
|
||||
from lbry.wallet import SQLiteMixin
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.dht.constants import DATA_EXPIRATION
|
||||
from lbry.blob.blob_info import BlobInfo
|
||||
@@ -112,7 +112,7 @@ def _batched_select(transaction, query, parameters, batch_size=900):
|
||||
|
||||
def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
|
||||
raw_content_fee, fully_reflected):
|
||||
raw_content_fee):
|
||||
return {
|
||||
"rowid": rowid,
|
||||
"added_on": added_on,
|
||||
@@ -129,8 +129,7 @@ def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download
|
||||
"saved_file": bool(saved_file),
|
||||
"content_fee": None if not raw_content_fee else Transaction(
|
||||
binascii.unhexlify(raw_content_fee)
|
||||
),
|
||||
"fully_reflected": fully_reflected
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -139,14 +138,11 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
signed_claims = {}
|
||||
for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
|
||||
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
|
||||
"select file.rowid, file.*, stream.*, c.*, "
|
||||
" case when (SELECT 1 FROM reflected_stream r WHERE r.sd_hash=stream.sd_hash) "
|
||||
" is null then 0 else 1 end as fully_reflected "
|
||||
"select file.rowid, file.*, stream.*, c.* "
|
||||
"from file inner join stream on file.stream_hash=stream.stream_hash "
|
||||
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
|
||||
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
|
||||
"order by c.rowid desc").fetchall():
|
||||
claim_args, fully_reflected = tuple(claim_args[:-1]), claim_args[-1]
|
||||
claim = StoredContentClaim(*claim_args)
|
||||
if claim.channel_claim_id:
|
||||
if claim.channel_claim_id not in signed_claims:
|
||||
@@ -156,7 +152,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
|
||||
_get_lbry_file_stream_dict(
|
||||
rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
|
||||
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
|
||||
raw_content_fee, fully_reflected
|
||||
raw_content_fee
|
||||
)
|
||||
)
|
||||
for claim_name, claim_id in _batched_select(
|
||||
@@ -727,19 +723,6 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if claim_id_to_supports:
|
||||
await self.save_supports(claim_id_to_supports)
|
||||
|
||||
def save_claim_from_output(self, ledger, *outputs: Output):
|
||||
return self.save_claims([{
|
||||
"claim_id": output.claim_id,
|
||||
"name": output.claim_name,
|
||||
"amount": dewies_to_lbc(output.amount),
|
||||
"address": output.get_address(ledger),
|
||||
"txid": output.tx_ref.id,
|
||||
"nout": output.position,
|
||||
"value": output.claim,
|
||||
"height": output.tx_ref.height,
|
||||
"claim_sequence": -1,
|
||||
} for output in outputs])
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = {}
|
||||
for info in claim_infos:
|
||||
@@ -753,8 +736,7 @@ class SQLiteStorage(SQLiteMixin):
|
||||
return self.save_claims(to_save.values())
|
||||
|
||||
@staticmethod
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash=None, bt_infohash=None):
|
||||
assert stream_hash or bt_infohash
|
||||
def _save_content_claim(transaction, claim_outpoint, stream_hash):
|
||||
# get the claim id and serialized metadata
|
||||
claim_info = transaction.execute(
|
||||
"select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,)
|
||||
@@ -802,19 +784,6 @@ class SQLiteStorage(SQLiteMixin):
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
await self.content_claim_callbacks[stream_hash]()
|
||||
|
||||
async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name):
|
||||
def _save_torrent(transaction):
|
||||
transaction.execute(
|
||||
"insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name)
|
||||
).fetchall()
|
||||
transaction.execute(
|
||||
"insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint)
|
||||
).fetchall()
|
||||
await self.db.run(_save_torrent)
|
||||
# update corresponding ManagedEncryptedFileDownloader object
|
||||
if bt_infohash in self.content_claim_callbacks:
|
||||
await self.content_claim_callbacks[bt_infohash]()
|
||||
|
||||
async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict:
|
||||
claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash])
|
||||
claim = None
|
||||
@@ -826,10 +795,6 @@ class SQLiteStorage(SQLiteMixin):
|
||||
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
|
||||
return claim
|
||||
|
||||
async def get_content_claim_for_torrent(self, bt_infohash):
|
||||
claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash])
|
||||
return claims[bt_infohash].as_dict() if claims else None
|
||||
|
||||
# # # # # # # # # reflector functions # # # # # # # # #
|
||||
|
||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||
|
||||
Whitespace-only changes.
@@ -1,302 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError
|
||||
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
from lbry.torrent.torrent_manager import TorrentSource
|
||||
from lbry.utils import cache_concurrent
|
||||
from lbry.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.wallet import WalletManager, Output
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FileManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'WalletManager',
|
||||
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.wallet_manager = wallet_manager
|
||||
self.storage = storage
|
||||
self.analytics_manager = analytics_manager
|
||||
self.source_managers: typing.Dict[str, SourceManager] = {}
|
||||
self.started = asyncio.Event()
|
||||
|
||||
@property
|
||||
def streams(self):
|
||||
return self.source_managers['stream']._sources
|
||||
|
||||
async def create_stream(self, file_path: str, key: Optional[bytes] = None, **kwargs) -> ManagedDownloadSource:
|
||||
if 'stream' in self.source_managers:
|
||||
return await self.source_managers['stream'].create(file_path, key, **kwargs)
|
||||
raise NotImplementedError
|
||||
|
||||
async def start(self):
|
||||
await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values()))
|
||||
for manager in self.source_managers.values():
|
||||
await manager.started.wait()
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
for manager in self.source_managers.values():
|
||||
# fixme: pop or not?
|
||||
manager.stop()
|
||||
self.started.clear()
|
||||
|
||||
@cache_concurrent
|
||||
async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
timeout: Optional[float] = None, file_name: Optional[str] = None,
|
||||
download_directory: Optional[str] = None,
|
||||
save_file: Optional[bool] = None, resolve_timeout: float = 3.0,
|
||||
wallet: Optional['Wallet'] = None) -> ManagedDownloadSource:
|
||||
|
||||
wallet = wallet or self.wallet_manager.default_wallet
|
||||
timeout = timeout or self.config.download_timeout
|
||||
start_time = self.loop.time()
|
||||
resolved_time = None
|
||||
stream = None
|
||||
claim = None
|
||||
error = None
|
||||
outpoint = None
|
||||
if save_file is None:
|
||||
save_file = self.config.save_files
|
||||
if file_name and not save_file:
|
||||
save_file = True
|
||||
if save_file:
|
||||
download_directory = download_directory or self.config.download_dir
|
||||
else:
|
||||
download_directory = None
|
||||
|
||||
payment = None
|
||||
try:
|
||||
# resolve the claim
|
||||
if not URL.parse(uri).has_stream:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
try:
|
||||
resolved_result = await asyncio.wait_for(
|
||||
self.wallet_manager.ledger.resolve(
|
||||
wallet.accounts, [uri],
|
||||
include_purchase_receipt=True,
|
||||
include_is_my_output=True
|
||||
), resolve_timeout
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
raise
|
||||
log.exception("Unexpected error resolving stream:")
|
||||
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
||||
if 'error' in resolved_result:
|
||||
raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}")
|
||||
if not resolved_result or uri not in resolved_result:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
txo = resolved_result[uri]
|
||||
if isinstance(txo, dict):
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}': {txo}")
|
||||
claim = txo.claim
|
||||
outpoint = f"{txo.tx_ref.id}:{txo.position}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
await self.storage.save_claim_from_output(self.wallet_manager.ledger, txo)
|
||||
|
||||
####################
|
||||
# update or replace
|
||||
####################
|
||||
|
||||
if claim.stream.source.bt_infohash:
|
||||
source_manager = self.source_managers['torrent']
|
||||
existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash)
|
||||
elif claim.stream.source.sd_hash:
|
||||
source_manager = self.source_managers['stream']
|
||||
existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash)
|
||||
else:
|
||||
raise ResolveError(f"There is nothing to download at {uri} - Source is unknown or unset")
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
to_replace, updated_stream = None, None
|
||||
if existing and existing[0].claim_id != txo.claim_id:
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}")
|
||||
if existing:
|
||||
log.info("claim contains a metadata only update to a stream we have")
|
||||
if claim.stream.source.bt_infohash:
|
||||
await self.storage.save_torrent_content_claim(
|
||||
existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name
|
||||
)
|
||||
claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier)
|
||||
existing[0].set_claim(claim_info, claim)
|
||||
else:
|
||||
await self.storage.save_content_claim(
|
||||
existing[0].stream_hash, outpoint
|
||||
)
|
||||
await source_manager._update_content_claim(existing[0])
|
||||
updated_stream = existing[0]
|
||||
else:
|
||||
existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id)
|
||||
if existing_for_claim_id:
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
if save_file and existing_for_claim_id[0].output_file_exists:
|
||||
save_file = False
|
||||
if not claim.stream.source.bt_infohash:
|
||||
existing_for_claim_id[0].downloader.node = source_manager.node
|
||||
await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file)
|
||||
if not existing_for_claim_id[0].output_file_exists and (
|
||||
save_file or file_name or download_directory):
|
||||
await existing_for_claim_id[0].save_file(
|
||||
file_name=file_name, download_directory=download_directory
|
||||
)
|
||||
to_replace = existing_for_claim_id[0]
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
if updated_stream:
|
||||
log.info("already have stream for %s", uri)
|
||||
if save_file and updated_stream.output_file_exists:
|
||||
save_file = False
|
||||
if not claim.stream.source.bt_infohash:
|
||||
updated_stream.downloader.node = source_manager.node
|
||||
await updated_stream.start(timeout=timeout, save_now=save_file)
|
||||
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
||||
await updated_stream.save_file(
|
||||
file_name=file_name, download_directory=download_directory
|
||||
)
|
||||
return updated_stream
|
||||
|
||||
####################
|
||||
# pay fee
|
||||
####################
|
||||
|
||||
needs_purchasing = (
|
||||
not to_replace and
|
||||
not txo.is_my_output and
|
||||
txo.has_price and
|
||||
not txo.purchase_receipt
|
||||
)
|
||||
|
||||
if needs_purchasing:
|
||||
payment = await self.wallet_manager.create_purchase_transaction(
|
||||
wallet.accounts, txo, exchange_rate_manager
|
||||
)
|
||||
|
||||
####################
|
||||
# make downloader and wait for start
|
||||
####################
|
||||
|
||||
if not claim.stream.source.bt_infohash:
|
||||
# fixme: this shouldnt be here
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash,
|
||||
download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
||||
analytics_manager=self.analytics_manager
|
||||
)
|
||||
stream.downloader.node = source_manager.node
|
||||
else:
|
||||
stream = TorrentSource(
|
||||
self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash,
|
||||
file_name=file_name, download_directory=download_directory or self.config.download_dir,
|
||||
status=ManagedStream.STATUS_RUNNING,
|
||||
analytics_manager=self.analytics_manager,
|
||||
torrent_session=source_manager.torrent_session
|
||||
)
|
||||
log.info("starting download for %s", uri)
|
||||
|
||||
before_download = self.loop.time()
|
||||
await stream.start(timeout, save_file)
|
||||
|
||||
####################
|
||||
# success case: delete to_replace if applicable, broadcast fee payment
|
||||
####################
|
||||
|
||||
if to_replace: # delete old stream now that the replacement has started downloading
|
||||
await source_manager.delete(to_replace)
|
||||
|
||||
if payment is not None:
|
||||
await self.wallet_manager.broadcast_or_release(payment)
|
||||
payment = None # to avoid releasing in `finally` later
|
||||
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
|
||||
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
||||
|
||||
source_manager.add(stream)
|
||||
|
||||
if not claim.stream.source.bt_infohash:
|
||||
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
||||
else:
|
||||
await self.storage.save_torrent_content_claim(
|
||||
stream.identifier, outpoint, stream.torrent_length, stream.torrent_name
|
||||
)
|
||||
claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier)
|
||||
stream.set_claim(claim_info, claim)
|
||||
if save_file:
|
||||
await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download),
|
||||
loop=self.loop)
|
||||
return stream
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||
KeyFeeAboveMaxAllowedError)
|
||||
if isinstance(err, expected):
|
||||
log.warning("Failed to download %s: %s", uri, str(err))
|
||||
elif isinstance(err, asyncio.CancelledError):
|
||||
pass
|
||||
else:
|
||||
log.exception("Unexpected error downloading stream:")
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
if payment is not None:
|
||||
# payment is set to None after broadcasting, if we're here an exception probably happened
|
||||
await self.wallet_manager.ledger.release_tx(payment)
|
||||
if self.analytics_manager and claim and claim.stream.source.bt_infohash:
|
||||
# TODO: analytics for torrents
|
||||
pass
|
||||
elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
||||
stream.downloader.time_to_first_bytes))):
|
||||
server = self.wallet_manager.ledger.network.client.server
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
||||
uri, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
|
||||
False if not stream else stream.downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
||||
None if not stream else stream.sd_hash,
|
||||
None if not stream else stream.downloader.time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
None if not stream else stream.downloader.time_to_first_bytes,
|
||||
None if not error else error.__class__.__name__,
|
||||
None if not error else str(error),
|
||||
None if not server else f"{server[0]}:{server[1]}"
|
||||
)
|
||||
)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
return await self.source_managers['stream'].stream_partial_content(request, sd_hash)
|
||||
|
||||
def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), [])
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file=False):
|
||||
for manager in self.source_managers.values():
|
||||
await manager.delete(source, delete_file)
|
||||
@@ -1,161 +0,0 @@
|
||||
import os
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
import binascii
|
||||
from typing import Optional
|
||||
from lbry.utils import generate_id
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ManagedDownloadSource:
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
SAVING_ID = 1
|
||||
STREAMING_ID = 2
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
|
||||
file_name: Optional[str] = None, download_directory: Optional[str] = None,
|
||||
status: Optional[str] = STATUS_STOPPED, claim: Optional[StoredContentClaim] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None):
|
||||
self.loop = loop
|
||||
self.storage = storage
|
||||
self.config = config
|
||||
self.identifier = identifier
|
||||
self.download_directory = download_directory
|
||||
self._file_name = file_name
|
||||
self._status = status
|
||||
self.stream_claim_info = claim
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.rowid = rowid
|
||||
self.content_fee = content_fee
|
||||
self.purchase_receipt = None
|
||||
self._added_on = added_on
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.saving = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=self.loop)
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||
|
||||
# @classmethod
|
||||
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str,
|
||||
# key: Optional[bytes] = None,
|
||||
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
|
||||
# raise NotImplementedError()
|
||||
|
||||
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def stop(self, finished: bool = False):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stop_tasks(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
||||
self.stream_claim_info = StoredContentClaim(
|
||||
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
|
||||
# async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||
# if not claim_info:
|
||||
# claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||
# self.set_claim(claim_info, claim_info['value'])
|
||||
|
||||
@property
|
||||
def file_name(self) -> Optional[str]:
|
||||
return self._file_name
|
||||
|
||||
@property
|
||||
def added_on(self) -> Optional[int]:
|
||||
return self._added_on
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
|
||||
@property
|
||||
def completed(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
# @property
|
||||
# def stream_url(self):
|
||||
# return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash}
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self.status == self.STATUS_FINISHED
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.status == self.STATUS_RUNNING
|
||||
|
||||
@property
|
||||
def claim_id(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
|
||||
|
||||
@property
|
||||
def txid(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.txid
|
||||
|
||||
@property
|
||||
def nout(self) -> Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.nout
|
||||
|
||||
@property
|
||||
def outpoint(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
|
||||
|
||||
@property
|
||||
def claim_height(self) -> Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.height
|
||||
|
||||
@property
|
||||
def channel_claim_id(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
|
||||
|
||||
@property
|
||||
def channel_name(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
|
||||
|
||||
@property
|
||||
def claim_name(self) -> Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||
|
||||
@property
|
||||
def metadata(self) -> Optional[typing.Dict]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||
|
||||
@property
|
||||
def metadata_protobuf(self) -> bytes:
|
||||
if self.stream_claim_info:
|
||||
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
||||
|
||||
@property
|
||||
def full_path(self) -> Optional[str]:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
||||
if self.file_name and self.download_directory else None
|
||||
|
||||
@property
|
||||
def output_file_exists(self):
|
||||
return os.path.isfile(self.full_path) if self.full_path else False
|
||||
@@ -1,138 +0,0 @@
|
||||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
from typing import Optional
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
COMPARISON_OPERATORS = {
|
||||
'eq': lambda a, b: a == b,
|
||||
'ne': lambda a, b: a != b,
|
||||
'g': lambda a, b: a > b,
|
||||
'l': lambda a, b: a < b,
|
||||
'ge': lambda a, b: a >= b,
|
||||
'le': lambda a, b: a <= b,
|
||||
}
|
||||
|
||||
|
||||
class SourceManager:
|
||||
filter_fields = {
|
||||
'rowid',
|
||||
'status',
|
||||
'file_name',
|
||||
'added_on',
|
||||
'download_path',
|
||||
'claim_name',
|
||||
'claim_height',
|
||||
'claim_id',
|
||||
'outpoint',
|
||||
'txid',
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name',
|
||||
'completed'
|
||||
}
|
||||
|
||||
set_filter_fields = {
|
||||
"claim_ids": "claim_id",
|
||||
"channel_claim_ids": "channel_claim_id",
|
||||
"outpoints": "outpoint"
|
||||
}
|
||||
|
||||
source_class = ManagedDownloadSource
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage',
|
||||
analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.storage = storage
|
||||
self.analytics_manager = analytics_manager
|
||||
self._sources: typing.Dict[str, ManagedDownloadSource] = {}
|
||||
self.started = asyncio.Event(loop=self.loop)
|
||||
|
||||
def add(self, source: ManagedDownloadSource):
|
||||
self._sources[source.identifier] = source
|
||||
|
||||
def remove(self, source: ManagedDownloadSource):
|
||||
if source.identifier not in self._sources:
|
||||
return
|
||||
self._sources.pop(source.identifier)
|
||||
source.stop_tasks()
|
||||
|
||||
async def initialize_from_database(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def start(self):
|
||||
await self.initialize_from_database()
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
while self._sources:
|
||||
_, source = self._sources.popitem()
|
||||
source.stop_tasks()
|
||||
self.started.clear()
|
||||
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedDownloadSource:
|
||||
raise NotImplementedError()
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
self.remove(source)
|
||||
if delete_file and source.output_file_exists:
|
||||
os.remove(source.full_path)
|
||||
|
||||
def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
|
||||
comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
if sort_by and sort_by not in self.filter_fields:
|
||||
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
||||
if comparison and comparison not in COMPARISON_OPERATORS:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
|
||||
for search in search_by:
|
||||
if search not in self.filter_fields:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
|
||||
compare_sets = {}
|
||||
if isinstance(search_by.get('claim_id'), list):
|
||||
compare_sets['claim_ids'] = search_by.pop('claim_id')
|
||||
if isinstance(search_by.get('outpoint'), list):
|
||||
compare_sets['outpoints'] = search_by.pop('outpoint')
|
||||
if isinstance(search_by.get('channel_claim_id'), list):
|
||||
compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id')
|
||||
|
||||
if search_by or compare_sets:
|
||||
comparison = comparison or 'eq'
|
||||
streams = []
|
||||
for stream in self._sources.values():
|
||||
if compare_sets and not all(
|
||||
getattr(stream, self.set_filter_fields[set_search]) in val
|
||||
for set_search, val in compare_sets.items()):
|
||||
continue
|
||||
if search_by and not all(
|
||||
COMPARISON_OPERATORS[comparison](getattr(stream, search), val)
|
||||
for search, val in search_by.items()):
|
||||
continue
|
||||
streams.append(stream)
|
||||
else:
|
||||
streams = list(self._sources.values())
|
||||
if sort_by:
|
||||
streams.sort(key=lambda s: getattr(s, sort_by) or "")
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
@@ -1,456 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import platform
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
from math import ceil
|
||||
|
||||
import lbry.utils
|
||||
from lbry.conf import TranscodeConfig
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class VideoFileAnalyzer:
|
||||
|
||||
def _replace_or_pop_env(self, variable):
|
||||
if variable + '_ORIG' in self._env_copy:
|
||||
self._env_copy[variable] = self._env_copy[variable + '_ORIG']
|
||||
else:
|
||||
self._env_copy.pop(variable, None)
|
||||
|
||||
def __init__(self, conf: TranscodeConfig):
|
||||
self._conf = conf
|
||||
self._available_encoders = ""
|
||||
self._ffmpeg_installed = None
|
||||
self._which_ffmpeg = None
|
||||
self._which_ffprobe = None
|
||||
self._env_copy = dict(os.environ)
|
||||
self._checked_ffmpeg = False
|
||||
if lbry.utils.is_running_from_bundle():
|
||||
# handle the situation where PyInstaller overrides our runtime environment:
|
||||
self._replace_or_pop_env('LD_LIBRARY_PATH')
|
||||
|
||||
@staticmethod
|
||||
def _execute(command, environment):
|
||||
# log.debug("Executing: %s", command)
|
||||
try:
|
||||
with subprocess.Popen(
|
||||
shlex.split(command) if platform.system() != 'Windows' else command,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment
|
||||
) as process:
|
||||
(stdout, stderr) = process.communicate() # blocks until the process exits
|
||||
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
|
||||
except subprocess.SubprocessError as e:
|
||||
return str(e), -1
|
||||
|
||||
# This create_subprocess_exec call is broken in Windows Python 3.7, but it's prettier than what's here.
|
||||
# The recommended fix is switching to ProactorEventLoop, but that breaks UDP in Linux Python 3.7.
|
||||
# We work around that issue here by using run_in_executor. Check it again in Python 3.8.
|
||||
async def _execute_ffmpeg(self, arguments):
|
||||
arguments = self._which_ffmpeg + " " + arguments
|
||||
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
|
||||
|
||||
async def _execute_ffprobe(self, arguments):
|
||||
arguments = self._which_ffprobe + " " + arguments
|
||||
return await asyncio.get_event_loop().run_in_executor(None, self._execute, arguments, self._env_copy)
|
||||
|
||||
async def _verify_executables(self):
|
||||
try:
|
||||
await self._execute_ffprobe("-version")
|
||||
version, code = await self._execute_ffmpeg("-version")
|
||||
except Exception as e:
|
||||
code = -1
|
||||
version = str(e)
|
||||
if code != 0 or not version.startswith("ffmpeg"):
|
||||
log.warning("Unable to run ffmpeg, but it was requested. Code: %d; Message: %s", code, version)
|
||||
raise FileNotFoundError(f"Unable to locate or run ffmpeg or ffprobe. Please install FFmpeg "
|
||||
f"and ensure that it is callable via PATH or conf.ffmpeg_path")
|
||||
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which_ffmpeg)
|
||||
return version
|
||||
|
||||
@staticmethod
|
||||
def _which_ffmpeg_and_ffmprobe(path):
|
||||
return shutil.which("ffmpeg", path=path), shutil.which("ffprobe", path=path)
|
||||
|
||||
async def _verify_ffmpeg_installed(self):
|
||||
if self._ffmpeg_installed:
|
||||
return
|
||||
self._ffmpeg_installed = False
|
||||
path = self._conf.ffmpeg_path
|
||||
if hasattr(self._conf, "data_dir"):
|
||||
path += os.path.pathsep + os.path.join(getattr(self._conf, "data_dir"), "ffmpeg", "bin")
|
||||
path += os.path.pathsep + self._env_copy.get("PATH", "")
|
||||
self._which_ffmpeg, self._which_ffprobe = await asyncio.get_running_loop().run_in_executor(
|
||||
None, self._which_ffmpeg_and_ffmprobe, path
|
||||
)
|
||||
if not self._which_ffmpeg:
|
||||
log.warning("Unable to locate ffmpeg executable. Path: %s", path)
|
||||
raise FileNotFoundError(f"Unable to locate ffmpeg executable. Path: {path}")
|
||||
if not self._which_ffprobe:
|
||||
log.warning("Unable to locate ffprobe executable. Path: %s", path)
|
||||
raise FileNotFoundError(f"Unable to locate ffprobe executable. Path: {path}")
|
||||
if os.path.dirname(self._which_ffmpeg) != os.path.dirname(self._which_ffprobe):
|
||||
log.warning("ffmpeg and ffprobe are in different folders!")
|
||||
|
||||
await self._verify_executables()
|
||||
self._ffmpeg_installed = True
|
||||
|
||||
async def status(self, reset=False, recheck=False):
|
||||
if reset:
|
||||
self._available_encoders = ""
|
||||
self._ffmpeg_installed = None
|
||||
if self._checked_ffmpeg and not recheck:
|
||||
pass
|
||||
elif self._ffmpeg_installed is None:
|
||||
try:
|
||||
await self._verify_ffmpeg_installed()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
self._checked_ffmpeg = True
|
||||
return {
|
||||
"available": self._ffmpeg_installed,
|
||||
"which": self._which_ffmpeg,
|
||||
"analyze_audio_volume": int(self._conf.volume_analysis_time) > 0
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _verify_container(scan_data: json):
|
||||
container = scan_data["format"]["format_name"]
|
||||
log.debug(" Detected container is %s", container)
|
||||
splits = container.split(",")
|
||||
if not {"webm", "mp4", "3gp", "ogg"}.intersection(splits):
|
||||
return "Container format is not in the approved list of WebM, MP4. " \
|
||||
f"Actual: {container} [{scan_data['format']['format_long_name']}]"
|
||||
|
||||
if "matroska" in splits:
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] == "video":
|
||||
codec = stream["codec_name"]
|
||||
if not {"vp8", "vp9", "av1"}.intersection(codec.split(",")):
|
||||
return "WebM format requires VP8/9 or AV1 video. " \
|
||||
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||
elif stream["codec_type"] == "audio":
|
||||
codec = stream["codec_name"]
|
||||
if not {"vorbis", "opus"}.intersection(codec.split(",")):
|
||||
return "WebM format requires Vorbis or Opus audio. " \
|
||||
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _verify_video_encoding(scan_data: json):
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] != "video":
|
||||
continue
|
||||
codec = stream["codec_name"]
|
||||
log.debug(" Detected video codec is %s, format is %s", codec, stream["pix_fmt"])
|
||||
if not {"h264", "vp8", "vp9", "av1", "theora"}.intersection(codec.split(",")):
|
||||
return "Video codec is not in the approved list of H264, VP8, VP9, AV1, Theora. " \
|
||||
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||
|
||||
if "h264" in codec.split(",") and stream["pix_fmt"] != "yuv420p":
|
||||
return "Video codec is H264, but its pixel format does not match the approved yuv420p. " \
|
||||
f"Actual: {stream['pix_fmt']}"
|
||||
|
||||
return ""
|
||||
|
||||
def _verify_bitrate(self, scan_data: json, file_path):
|
||||
bit_rate_max = float(self._conf.video_bitrate_maximum)
|
||||
if bit_rate_max <= 0:
|
||||
return ""
|
||||
|
||||
if "bit_rate" in scan_data["format"]:
|
||||
bit_rate = float(scan_data["format"]["bit_rate"])
|
||||
else:
|
||||
bit_rate = os.stat(file_path).st_size / float(scan_data["format"]["duration"])
|
||||
log.debug(" Detected bitrate is %s Mbps. Allowed max: %s Mbps",
|
||||
str(bit_rate / 1000000.0), str(bit_rate_max / 1000000.0))
|
||||
|
||||
if bit_rate > bit_rate_max:
|
||||
return "The bit rate is above the configured maximum. Actual: " \
|
||||
f"{bit_rate / 1000000.0} Mbps; Allowed max: {bit_rate_max / 1000000.0} Mbps"
|
||||
|
||||
return ""
|
||||
|
||||
async def _verify_fast_start(self, scan_data: json, video_file):
|
||||
container = scan_data["format"]["format_name"]
|
||||
if {"webm", "ogg"}.intersection(container.split(",")):
|
||||
return ""
|
||||
|
||||
result, _ = await self._execute_ffprobe(f'-v debug "{video_file}"')
|
||||
match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
|
||||
if match and int(match.group(1)) != 0:
|
||||
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _verify_audio_encoding(scan_data: json):
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] != "audio":
|
||||
continue
|
||||
codec = stream["codec_name"]
|
||||
log.debug(" Detected audio codec is %s", codec)
|
||||
if not {"aac", "mp3", "flac", "vorbis", "opus"}.intersection(codec.split(",")):
|
||||
return "Audio codec is not in the approved list of AAC, FLAC, MP3, Vorbis, and Opus. " \
|
||||
f"Actual: {codec} [{stream['codec_long_name']}]"
|
||||
if int(stream['sample_rate']) > 48000:
|
||||
return "Sample rate out of range"
|
||||
|
||||
return ""
|
||||
|
||||
async def _verify_audio_volume(self, seconds, video_file):
|
||||
try:
|
||||
validate_volume = int(seconds) > 0
|
||||
except ValueError:
|
||||
validate_volume = False
|
||||
|
||||
if not validate_volume:
|
||||
return ""
|
||||
|
||||
result, _ = await self._execute_ffmpeg(f'-i "{video_file}" -t {seconds} '
|
||||
f'-af volumedetect -vn -sn -dn -f null "{os.devnull}"')
|
||||
try:
|
||||
mean_volume = float(re.search(r"mean_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
|
||||
max_volume = float(re.search(r"max_volume:\s+([-+]?\d*\.\d+|\d+)", result).group(1))
|
||||
except Exception as e:
|
||||
log.debug(" Failure in volume analysis. Message: %s", str(e))
|
||||
return ""
|
||||
|
||||
if max_volume < -5.0 and mean_volume < -22.0:
|
||||
return "Audio is at least five dB lower than prime. " \
|
||||
f"Actual max: {max_volume}, mean: {mean_volume}"
|
||||
|
||||
log.debug(" Detected audio volume has mean, max of %f, %f dB", mean_volume, max_volume)
|
||||
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _compute_crf(scan_data):
|
||||
height = 240.0
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] == "video":
|
||||
height = max(height, float(stream["height"]))
|
||||
|
||||
# https://developers.google.com/media/vp9/settings/vod/
|
||||
return int(-0.011 * height + 40)
|
||||
|
||||
def _get_video_scaler(self):
|
||||
return self._conf.video_scaler
|
||||
|
||||
async def _get_video_encoder(self, scan_data):
|
||||
# use what the user said if it's there:
|
||||
# if it's not there, use h264 if we can because it's way faster than the others
|
||||
# if we don't have h264 use vp9; it's fairly compatible even though it's slow
|
||||
|
||||
if not self._available_encoders:
|
||||
self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
|
||||
|
||||
encoder = self._conf.video_encoder.split(" ", 1)[0]
|
||||
if re.search(fr"^\s*V..... {encoder} ", self._available_encoders, re.MULTILINE):
|
||||
return self._conf.video_encoder
|
||||
|
||||
if re.search(r"^\s*V..... libx264 ", self._available_encoders, re.MULTILINE):
|
||||
if encoder:
|
||||
log.warning(" Using libx264 since the requested encoder was unavailable. Requested: %s", encoder)
|
||||
return 'libx264 -crf 19 -vf "format=yuv420p"'
|
||||
|
||||
if not encoder:
|
||||
encoder = "libx264"
|
||||
|
||||
if re.search(r"^\s*V..... libvpx-vp9 ", self._available_encoders, re.MULTILINE):
|
||||
log.warning(" Using libvpx-vp9 since the requested encoder was unavailable. Requested: %s", encoder)
|
||||
crf = self._compute_crf(scan_data)
|
||||
return f"libvpx-vp9 -crf {crf} -b:v 0"
|
||||
|
||||
if re.search(r"^\s*V..... libtheora", self._available_encoders, re.MULTILINE):
|
||||
log.warning(" Using libtheora since the requested encoder was unavailable. Requested: %s", encoder)
|
||||
return "libtheora -q:v 7"
|
||||
|
||||
raise Exception(f"The video encoder is not available. Requested: {encoder}")
|
||||
|
||||
async def _get_audio_encoder(self, extension):
|
||||
# if the video encoding is theora or av1/vp8/vp9 use opus (or fallback to vorbis)
|
||||
# or we don't have a video encoding but we have an ogg or webm container use opus
|
||||
# if we need to use opus/vorbis see if the conf file has it else use our own params
|
||||
# else use the user-set value if it exists
|
||||
# else use aac
|
||||
|
||||
wants_opus = extension != "mp4"
|
||||
if not self._available_encoders:
|
||||
self._available_encoders, _ = await self._execute_ffmpeg("-encoders -v quiet")
|
||||
|
||||
encoder = self._conf.audio_encoder.split(" ", 1)[0]
|
||||
if wants_opus and 'opus' in encoder:
|
||||
return self._conf.audio_encoder
|
||||
|
||||
if wants_opus and re.search(r"^\s*A..... libopus ", self._available_encoders, re.MULTILINE):
|
||||
return "libopus -b:a 160k"
|
||||
|
||||
if wants_opus and 'vorbis' in encoder:
|
||||
return self._conf.audio_encoder
|
||||
|
||||
if wants_opus and re.search(r"^\s*A..... libvorbis ", self._available_encoders, re.MULTILINE):
|
||||
return "libvorbis -q:a 6"
|
||||
|
||||
if re.search(fr"^\s*A..... {encoder} ", self._available_encoders, re.MULTILINE):
|
||||
return self._conf.audio_encoder
|
||||
|
||||
if re.search(r"^\s*A..... aac ", self._available_encoders, re.MULTILINE):
|
||||
return "aac -b:a 192k"
|
||||
|
||||
raise Exception(f"The audio encoder is not available. Requested: {encoder or 'aac'}")
|
||||
|
||||
@staticmethod
|
||||
def _get_best_container_extension(scan_data, video_encoder):
|
||||
# the container is chosen by the video format
|
||||
# if we are theora-encoded, we want ogg
|
||||
# if we are vp8/vp9/av1 we want webm
|
||||
# use mp4 for anything else
|
||||
|
||||
if video_encoder: # not re-encoding video
|
||||
if "theora" in video_encoder:
|
||||
return "ogv"
|
||||
if re.search(r"vp[89x]|av1", video_encoder.split(" ", 1)[0]):
|
||||
return "webm"
|
||||
return "mp4"
|
||||
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] != "video":
|
||||
continue
|
||||
codec = stream["codec_name"].split(",")
|
||||
if "theora" in codec:
|
||||
return "ogv"
|
||||
if {"vp8", "vp9", "av1"}.intersection(codec):
|
||||
return "webm"
|
||||
|
||||
return "mp4"
|
||||
|
||||
async def _get_scan_data(self, validate, file_path):
|
||||
arguments = f'-v quiet -print_format json -show_format -show_streams "{file_path}"'
|
||||
result, _ = await self._execute_ffprobe(arguments)
|
||||
try:
|
||||
scan_data = json.loads(result)
|
||||
except Exception as e:
|
||||
log.debug("Failure in JSON parsing ffprobe results. Message: %s", str(e))
|
||||
raise ValueError(f'Absent or unreadable video file: {file_path}')
|
||||
|
||||
if "format" not in scan_data or "duration" not in scan_data["format"]:
|
||||
log.debug("Format data is missing from ffprobe results for: %s", file_path)
|
||||
raise ValueError(f'Media file does not appear to contain video content: {file_path}')
|
||||
|
||||
if float(scan_data["format"]["duration"]) < 0.1:
|
||||
log.debug("Media file appears to be an image: %s", file_path)
|
||||
raise ValueError(f'Assuming image file at: {file_path}')
|
||||
|
||||
return scan_data
|
||||
|
||||
@staticmethod
|
||||
def _build_spec(scan_data):
|
||||
assert scan_data
|
||||
|
||||
duration = ceil(float(scan_data["format"]["duration"])) # existence verified when scan_data made
|
||||
width = -1
|
||||
height = -1
|
||||
for stream in scan_data["streams"]:
|
||||
if stream["codec_type"] != "video":
|
||||
continue
|
||||
width = max(width, int(stream["width"]))
|
||||
height = max(height, int(stream["height"]))
|
||||
|
||||
log.debug(" Detected duration: %d sec. with resolution: %d x %d", duration, width, height)
|
||||
|
||||
spec = {"duration": duration}
|
||||
if height >= 0:
|
||||
spec["height"] = height
|
||||
if width >= 0:
|
||||
spec["width"] = width
|
||||
return spec
|
||||
|
||||
async def verify_or_repair(self, validate, repair, file_path, ignore_non_video=False):
|
||||
if not validate and not repair:
|
||||
return file_path, {}
|
||||
|
||||
if ignore_non_video and not file_path:
|
||||
return file_path, {}
|
||||
|
||||
await self._verify_ffmpeg_installed()
|
||||
try:
|
||||
scan_data = await self._get_scan_data(validate, file_path)
|
||||
except ValueError:
|
||||
if ignore_non_video:
|
||||
return file_path, {}
|
||||
raise
|
||||
|
||||
fast_start_msg = await self._verify_fast_start(scan_data, file_path)
|
||||
log.debug("Analyzing %s:", file_path)
|
||||
spec = self._build_spec(scan_data)
|
||||
log.debug(" Detected faststart is %s", "false" if fast_start_msg else "true")
|
||||
container_msg = self._verify_container(scan_data)
|
||||
bitrate_msg = self._verify_bitrate(scan_data, file_path)
|
||||
video_msg = self._verify_video_encoding(scan_data)
|
||||
audio_msg = self._verify_audio_encoding(scan_data)
|
||||
volume_msg = await self._verify_audio_volume(self._conf.volume_analysis_time, file_path)
|
||||
messages = [container_msg, bitrate_msg, fast_start_msg, video_msg, audio_msg, volume_msg]
|
||||
|
||||
if not any(messages):
|
||||
return file_path, spec
|
||||
|
||||
if not repair:
|
||||
errors = ["Streamability verification failed:"]
|
||||
errors.extend(filter(None, messages))
|
||||
raise Exception("\n ".join(errors))
|
||||
|
||||
# the plan for transcoding:
|
||||
# we have to re-encode the video if it is in a nonstandard format
|
||||
# we also re-encode if we are h264 but not yuv420p (both errors caught in video_msg)
|
||||
# we also re-encode if our bitrate or sample rate is too high
|
||||
|
||||
try:
|
||||
transcode_command = [f'-i "{file_path}" -y -c:s copy -c:d copy -c:v']
|
||||
|
||||
video_encoder = ""
|
||||
if video_msg or bitrate_msg:
|
||||
video_encoder = await self._get_video_encoder(scan_data)
|
||||
transcode_command.append(video_encoder)
|
||||
# could do the scaling only if bitrate_msg, but if we're going to the effort to re-encode anyway...
|
||||
transcode_command.append(self._get_video_scaler())
|
||||
else:
|
||||
transcode_command.append("copy")
|
||||
|
||||
transcode_command.append("-movflags +faststart -c:a")
|
||||
extension = self._get_best_container_extension(scan_data, video_encoder)
|
||||
|
||||
if audio_msg or volume_msg:
|
||||
audio_encoder = await self._get_audio_encoder(extension)
|
||||
transcode_command.append(audio_encoder)
|
||||
if volume_msg and self._conf.volume_filter:
|
||||
transcode_command.append(self._conf.volume_filter)
|
||||
if audio_msg == "Sample rate out of range":
|
||||
transcode_command.append(" -ar 48000 ")
|
||||
else:
|
||||
transcode_command.append("copy")
|
||||
|
||||
# TODO: put it in a temp folder and delete it after we upload?
|
||||
path = pathlib.Path(file_path)
|
||||
output = path.parent / f"{path.stem}_fixed.{extension}"
|
||||
transcode_command.append(f'"{output}"')
|
||||
|
||||
ffmpeg_command = " ".join(transcode_command)
|
||||
log.info("Proceeding on transcode via: ffmpeg %s", ffmpeg_command)
|
||||
result, code = await self._execute_ffmpeg(ffmpeg_command)
|
||||
if code != 0:
|
||||
raise Exception(f"Failure to complete the transcode command. Output: {result}")
|
||||
except Exception as e:
|
||||
if validate:
|
||||
raise
|
||||
log.info("Unable to transcode %s . Message: %s", file_path, str(e))
|
||||
# TODO: delete partial output file here if it exists?
|
||||
return file_path, spec
|
||||
|
||||
return str(output), spec
|
||||
@@ -1,68 +0,0 @@
|
||||
import time
|
||||
import logging
|
||||
import asyncio
|
||||
import asyncio.tasks
|
||||
from aiohttp import web
|
||||
from prometheus_client import generate_latest as prom_generate_latest
|
||||
from prometheus_client import Counter, Histogram, Gauge
|
||||
|
||||
|
||||
PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio')
|
||||
PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio')
|
||||
PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio')
|
||||
TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio')
|
||||
|
||||
|
||||
def get_loop_metrics(delay=1):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def callback(started):
|
||||
PROBE_TIMES.observe(time.perf_counter() - started - delay)
|
||||
PROBES_FINISHED.inc()
|
||||
|
||||
async def monitor_loop_responsiveness():
|
||||
while True:
|
||||
now = time.perf_counter()
|
||||
loop.call_later(delay, callback, now)
|
||||
PROBES_IN_FLIGHT.inc()
|
||||
TASK_COUNT.set(len(asyncio.tasks._all_tasks))
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
return loop.create_task(monitor_loop_responsiveness())
|
||||
|
||||
|
||||
class PrometheusServer:
|
||||
def __init__(self, logger=None):
|
||||
self.runner = None
|
||||
self.logger = logger or logging.getLogger(__name__)
|
||||
self._monitor_loop_task = None
|
||||
|
||||
async def start(self, interface: str, port: int):
|
||||
self.logger.info("start prometheus metrics")
|
||||
prom_app = web.Application()
|
||||
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
|
||||
self.runner = web.AppRunner(prom_app)
|
||||
await self.runner.setup()
|
||||
|
||||
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
|
||||
await metrics_site.start()
|
||||
self.logger.info(
|
||||
'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]
|
||||
)
|
||||
self._monitor_loop_task = get_loop_metrics()
|
||||
|
||||
async def handle_metrics_get_request(self, request: web.Request):
|
||||
try:
|
||||
return web.Response(
|
||||
text=prom_generate_latest().decode(),
|
||||
content_type='text/plain; version=0.0.4'
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception('could not generate prometheus data')
|
||||
raise
|
||||
|
||||
async def stop(self):
|
||||
if self._monitor_loop_task and not self._monitor_loop_task.done():
|
||||
self._monitor_loop_task.cancel()
|
||||
self._monitor_loop_task = None
|
||||
await self.runner.cleanup()
|
||||
@@ -1,24 +0,0 @@
|
||||
Schema
|
||||
=====
|
||||
|
||||
Those files are generated from the [types repo](https://github.com/lbryio/types). If you are modifying/adding a new type, make sure it is cloned in the same root folder as the SDK repo, like:
|
||||
|
||||
```
|
||||
repos/
|
||||
- lbry-sdk/
|
||||
- types/
|
||||
```
|
||||
|
||||
Then, [download protoc 3.2.0](https://github.com/protocolbuffers/protobuf/releases/tag/v3.2.0), add it to your PATH. On linux it is:
|
||||
|
||||
```bash
|
||||
cd ~/.local/bin
|
||||
wget https://github.com/protocolbuffers/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip
|
||||
unzip protoc-3.2.0-linux-x86_64.zip bin/protoc -d..
|
||||
```
|
||||
|
||||
Finally, `make` should update everything in place.
|
||||
|
||||
|
||||
### Why protoc 3.2.0?
|
||||
Different/newer versions will generate larger diffs and we need to make sure they are good. In theory, we can just update to latest and it will all work, but it is a good practice to check blockchain data and retro compatibility before bumping versions (if you do, please update this section!).
|
||||
@@ -30,10 +30,14 @@ class Claim(Signable):
|
||||
COLLECTION = 'collection'
|
||||
REPOST = 'repost'
|
||||
|
||||
__slots__ = ()
|
||||
__slots__ = 'version',
|
||||
|
||||
message_class = ClaimMessage
|
||||
|
||||
def __init__(self, message=None):
|
||||
super().__init__(message)
|
||||
self.version = 2
|
||||
|
||||
@property
|
||||
def claim_type(self) -> str:
|
||||
return self.message.WhichOneof('type')
|
||||
@@ -249,7 +253,7 @@ class Stream(BaseClaim):
|
||||
if stream_type in ('image', 'video', 'audio'):
|
||||
media = getattr(self, stream_type)
|
||||
media_args = {'file_metadata': None}
|
||||
if file_path is not None and not all((duration, width, height)):
|
||||
if file_path is not None:
|
||||
try:
|
||||
media_args['file_metadata'] = binary_file_metadata(binary_file_parser(file_path))
|
||||
except:
|
||||
@@ -303,10 +307,6 @@ class Stream(BaseClaim):
|
||||
def has_fee(self) -> bool:
|
||||
return self.message.HasField('fee')
|
||||
|
||||
@property
|
||||
def has_source(self) -> bool:
|
||||
return self.message.HasField('source')
|
||||
|
||||
@property
|
||||
def source(self) -> Source:
|
||||
return Source(self.message.source)
|
||||
|
||||
@@ -46,8 +46,8 @@ types_map = {
|
||||
'.ksh': ('text/plain', 'document'),
|
||||
'.latex': ('application/x-latex', 'binary'),
|
||||
'.m1v': ('video/mpeg', 'video'),
|
||||
'.m3u': ('application/x-mpegurl', 'audio'),
|
||||
'.m3u8': ('application/x-mpegurl', 'video'),
|
||||
'.m3u': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.m3u8': ('application/vnd.apple.mpegurl', 'audio'),
|
||||
'.man': ('application/x-troff-man', 'document'),
|
||||
'.markdown': ('text/markdown', 'document'),
|
||||
'.md': ('text/markdown', 'document'),
|
||||
@@ -61,12 +61,10 @@ types_map = {
|
||||
'.mp3': ('audio/mpeg', 'audio'),
|
||||
'.mp4': ('video/mp4', 'video'),
|
||||
'.mpa': ('video/mpeg', 'video'),
|
||||
'.mpd': ('application/dash+xml', 'video'),
|
||||
'.mpe': ('video/mpeg', 'video'),
|
||||
'.mpeg': ('video/mpeg', 'video'),
|
||||
'.mpg': ('video/mpeg', 'video'),
|
||||
'.ms': ('application/x-troff-ms', 'binary'),
|
||||
'.m4s': ('video/iso.segment', 'binary'),
|
||||
'.nc': ('application/x-netcdf', 'binary'),
|
||||
'.nws': ('message/rfc822', 'document'),
|
||||
'.o': ('application/octet-stream', 'binary'),
|
||||
@@ -120,12 +118,10 @@ types_map = {
|
||||
'.tif': ('image/tiff', 'image'),
|
||||
'.tiff': ('image/tiff', 'image'),
|
||||
'.tr': ('application/x-troff', 'binary'),
|
||||
'.ts': ('video/mp2t', 'video'),
|
||||
'.tsv': ('text/tab-separated-values', 'document'),
|
||||
'.txt': ('text/plain', 'document'),
|
||||
'.ustar': ('application/x-ustar', 'binary'),
|
||||
'.vcf': ('text/x-vcard', 'document'),
|
||||
'.vtt': ('text/vtt', 'document'),
|
||||
'.wav': ('audio/x-wav', 'audio'),
|
||||
'.webm': ('video/webm', 'video'),
|
||||
'.wiz': ('application/msword', 'document'),
|
||||
@@ -145,7 +141,6 @@ types_map = {
|
||||
'.cbz': ('application/vnd.comicbook+zip', 'document'),
|
||||
'.flac': ('audio/flac', 'audio'),
|
||||
'.lbry': ('application/x-ext-lbry', 'document'),
|
||||
'.m4a': ('audio/mp4', 'audio'),
|
||||
'.m4v': ('video/m4v', 'video'),
|
||||
'.mid': ('audio/midi', 'audio'),
|
||||
'.midi': ('audio/midi', 'audio'),
|
||||
@@ -153,7 +148,6 @@ types_map = {
|
||||
'.mobi': ('application/x-mobipocket-ebook', 'document'),
|
||||
'.oga': ('audio/ogg', 'audio'),
|
||||
'.ogv': ('video/ogg', 'video'),
|
||||
'.ogg': ('video/ogg', 'video'),
|
||||
'.pct': ('image/pict', 'image'),
|
||||
'.pic': ('image/pict', 'image'),
|
||||
'.pict': ('image/pict', 'image'),
|
||||
|
||||
+54
-70
@@ -4,54 +4,41 @@ from typing import List
|
||||
from binascii import hexlify
|
||||
from itertools import chain
|
||||
|
||||
from lbry.error import ResolveCensoredError
|
||||
from lbry.schema.types.v2.result_pb2 import Outputs as OutputsMessage
|
||||
from lbry.schema.types.v2.result_pb2 import Error as ErrorMessage
|
||||
|
||||
INVALID = ErrorMessage.Code.Name(ErrorMessage.INVALID)
|
||||
NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
|
||||
BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
|
||||
|
||||
|
||||
def set_reference(reference, txo_row):
|
||||
if txo_row:
|
||||
reference.tx_hash = txo_row['txo_hash'][:32]
|
||||
reference.nout = struct.unpack('<I', txo_row['txo_hash'][32:])[0]
|
||||
reference.height = txo_row['height']
|
||||
|
||||
|
||||
class Censor:
|
||||
|
||||
NOT_CENSORED = 0
|
||||
SEARCH = 1
|
||||
RESOLVE = 2
|
||||
__slots__ = 'streams', 'channels', 'censored', 'total'
|
||||
|
||||
__slots__ = 'censor_type', 'censored'
|
||||
|
||||
def __init__(self, censor_type):
|
||||
self.censor_type = censor_type
|
||||
def __init__(self, streams: dict = None, channels: dict = None):
|
||||
self.streams = streams or {}
|
||||
self.channels = channels or {}
|
||||
self.censored = {}
|
||||
|
||||
def is_censored(self, row):
|
||||
return (row.get('censor_type') or self.NOT_CENSORED) >= self.censor_type
|
||||
|
||||
def apply(self, rows):
|
||||
return [row for row in rows if not self.censor(row)]
|
||||
self.total = 0
|
||||
|
||||
def censor(self, row) -> bool:
|
||||
if self.is_censored(row):
|
||||
censoring_channel_hash = row['censoring_channel_hash']
|
||||
self.censored.setdefault(censoring_channel_hash, set())
|
||||
self.censored[censoring_channel_hash].add(row['tx_hash'])
|
||||
return True
|
||||
return False
|
||||
was_censored = False
|
||||
for claim_hash, lookup in (
|
||||
(row['claim_hash'], self.streams),
|
||||
(row['claim_hash'], self.channels),
|
||||
(row['channel_hash'], self.channels)):
|
||||
censoring_channel_hash = lookup.get(claim_hash)
|
||||
if censoring_channel_hash:
|
||||
was_censored = True
|
||||
self.censored.setdefault(censoring_channel_hash, 0)
|
||||
self.censored[censoring_channel_hash] += 1
|
||||
break
|
||||
if was_censored:
|
||||
self.total += 1
|
||||
return was_censored
|
||||
|
||||
def to_message(self, outputs: OutputsMessage, extra_txo_rows: dict):
|
||||
def to_message(self, outputs: OutputsMessage):
|
||||
outputs.blocked_total = self.total
|
||||
for censoring_channel_hash, count in self.censored.items():
|
||||
blocked = outputs.blocked.add()
|
||||
blocked.count = len(count)
|
||||
set_reference(blocked.channel, extra_txo_rows.get(censoring_channel_hash))
|
||||
outputs.blocked_total += len(count)
|
||||
block = outputs.blocked.add()
|
||||
block.count = count
|
||||
block.channel_hash = censoring_channel_hash
|
||||
|
||||
|
||||
class Outputs:
|
||||
@@ -73,35 +60,21 @@ class Outputs:
|
||||
for txo_message in self.extra_txos:
|
||||
self.message_to_txo(txo_message, tx_map)
|
||||
txos = [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos]
|
||||
return txos, self.inflate_blocked(tx_map)
|
||||
return txos, self.inflate_blocked()
|
||||
|
||||
def inflate_blocked(self, tx_map):
|
||||
def inflate_blocked(self):
|
||||
return {
|
||||
"total": self.blocked_total,
|
||||
"channels": [{
|
||||
'channel': self.message_to_txo(blocked.channel, tx_map),
|
||||
'blocked': blocked.count
|
||||
} for blocked in self.blocked]
|
||||
"channels": {
|
||||
hexlify(message.channel_hash[::-1]).decode(): message.count
|
||||
for message in self.blocked
|
||||
}
|
||||
}
|
||||
|
||||
def message_to_txo(self, txo_message, tx_map):
|
||||
if txo_message.WhichOneof('meta') == 'error':
|
||||
error = {
|
||||
'error': {
|
||||
'name': txo_message.error.Code.Name(txo_message.error.code),
|
||||
'text': txo_message.error.text,
|
||||
}
|
||||
}
|
||||
if error['error']['name'] == BLOCKED:
|
||||
error['error']['censor'] = self.message_to_txo(
|
||||
txo_message.error.blocked.channel, tx_map
|
||||
)
|
||||
return error
|
||||
|
||||
tx = tx_map.get(txo_message.tx_hash)
|
||||
if not tx:
|
||||
return
|
||||
txo = tx.outputs[txo_message.nout]
|
||||
return None
|
||||
txo = tx_map[txo_message.tx_hash].outputs[txo_message.nout]
|
||||
if txo_message.WhichOneof('meta') == 'claim':
|
||||
claim = txo_message.claim
|
||||
txo.meta = {
|
||||
@@ -156,30 +129,26 @@ class Outputs:
|
||||
|
||||
@classmethod
|
||||
def to_bytes(cls, txo_rows, extra_txo_rows, offset=0, total=None, blocked: Censor = None) -> bytes:
|
||||
extra_txo_rows = {row['claim_hash']: row for row in extra_txo_rows}
|
||||
page = OutputsMessage()
|
||||
page.offset = offset
|
||||
if total is not None:
|
||||
page.total = total
|
||||
if blocked is not None:
|
||||
blocked.to_message(page, extra_txo_rows)
|
||||
blocked.to_message(page)
|
||||
for row in txo_rows:
|
||||
cls.row_to_message(row, page.txos.add(), extra_txo_rows)
|
||||
for row in extra_txo_rows.values():
|
||||
for row in extra_txo_rows:
|
||||
cls.row_to_message(row, page.extra_txos.add(), extra_txo_rows)
|
||||
return page.SerializeToString()
|
||||
|
||||
@classmethod
|
||||
def row_to_message(cls, txo, txo_message, extra_row_dict: dict):
|
||||
def row_to_message(cls, txo, txo_message, extra_txo_rows):
|
||||
if isinstance(txo, Exception):
|
||||
txo_message.error.text = txo.args[0]
|
||||
if isinstance(txo, ValueError):
|
||||
txo_message.error.code = ErrorMessage.INVALID
|
||||
txo_message.error.code = txo_message.error.INVALID
|
||||
elif isinstance(txo, LookupError):
|
||||
txo_message.error.code = ErrorMessage.NOT_FOUND
|
||||
elif isinstance(txo, ResolveCensoredError):
|
||||
txo_message.error.code = ErrorMessage.BLOCKED
|
||||
set_reference(txo_message.error.blocked.channel, extra_row_dict.get(txo.censor_hash))
|
||||
txo_message.error.code = txo_message.error.NOT_FOUND
|
||||
return
|
||||
txo_message.tx_hash = txo['txo_hash'][:32]
|
||||
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
|
||||
@@ -202,5 +171,20 @@ class Outputs:
|
||||
txo_message.claim.trending_mixed = txo['trending_mixed']
|
||||
txo_message.claim.trending_local = txo['trending_local']
|
||||
txo_message.claim.trending_global = txo['trending_global']
|
||||
set_reference(txo_message.claim.channel, extra_row_dict.get(txo['channel_hash']))
|
||||
set_reference(txo_message.claim.repost, extra_row_dict.get(txo['reposted_claim_hash']))
|
||||
cls.set_reference(txo_message, 'channel', txo['channel_hash'], extra_txo_rows)
|
||||
cls.set_reference(txo_message, 'repost', txo['reposted_claim_hash'], extra_txo_rows)
|
||||
|
||||
@staticmethod
|
||||
def set_blocked(message, blocked):
|
||||
message.blocked_total = blocked.total
|
||||
|
||||
@staticmethod
|
||||
def set_reference(message, attr, claim_hash, rows):
|
||||
if claim_hash:
|
||||
for txo in rows:
|
||||
if claim_hash == txo['claim_hash']:
|
||||
reference = getattr(message.claim, attr)
|
||||
reference.tx_hash = txo['txo_hash'][:32]
|
||||
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
|
||||
reference.height = txo['height']
|
||||
break
|
||||
+1
-18
@@ -1,23 +1,6 @@
|
||||
from lbry.schema.base import Signable
|
||||
from lbry.schema.types.v2.support_pb2 import Support as SupportMessage
|
||||
|
||||
|
||||
class Support(Signable):
|
||||
__slots__ = ()
|
||||
message_class = SupportMessage
|
||||
|
||||
@property
|
||||
def emoji(self) -> str:
|
||||
return self.message.emoji
|
||||
|
||||
@emoji.setter
|
||||
def emoji(self, emoji: str):
|
||||
self.message.emoji = emoji
|
||||
|
||||
@property
|
||||
def comment(self) -> str:
|
||||
return self.message.comment
|
||||
|
||||
@comment.setter
|
||||
def comment(self, comment: str):
|
||||
self.message.comment = comment
|
||||
message_class = None # TODO: add support protobufs
|
||||
@@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='result.proto',
|
||||
package='pb',
|
||||
syntax='proto3',
|
||||
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.Outputb\x06proto3')
|
||||
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\".\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x14\n\x0c\x63hannel_hash\x18\x02 \x01(\x0c\x62\x06proto3')
|
||||
)
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
@@ -388,9 +388,9 @@ _BLOCKED = _descriptor.Descriptor(
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='channel', full_name='pb.Blocked.channel', index=1,
|
||||
number=2, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
name='channel_hash', full_name='pb.Blocked.channel_hash', index=1,
|
||||
number=2, type=12, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
@@ -407,7 +407,7 @@ _BLOCKED = _descriptor.Descriptor(
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=884,
|
||||
serialized_end=937,
|
||||
serialized_end=930,
|
||||
)
|
||||
|
||||
_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
|
||||
@@ -426,7 +426,6 @@ _CLAIMMETA.fields_by_name['repost'].message_type = _OUTPUT
|
||||
_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE
|
||||
_ERROR.fields_by_name['blocked'].message_type = _BLOCKED
|
||||
_ERROR_CODE.containing_type = _ERROR
|
||||
_BLOCKED.fields_by_name['channel'].message_type = _OUTPUT
|
||||
DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS
|
||||
DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT
|
||||
DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: support.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf import descriptor_pb2
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='support.proto',
|
||||
package='pb',
|
||||
syntax='proto3',
|
||||
serialized_pb=_b('\n\rsupport.proto\x12\x02pb\")\n\x07Support\x12\r\n\x05\x65moji\x18\x01 \x01(\t\x12\x0f\n\x07\x63omment\x18\x02 \x01(\tb\x06proto3')
|
||||
)
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
|
||||
|
||||
|
||||
_SUPPORT = _descriptor.Descriptor(
|
||||
name='Support',
|
||||
full_name='pb.Support',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='emoji', full_name='pb.Support.emoji', index=0,
|
||||
number=1, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='comment', full_name='pb.Support.comment', index=1,
|
||||
number=2, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=21,
|
||||
serialized_end=62,
|
||||
)
|
||||
|
||||
DESCRIPTOR.message_types_by_name['Support'] = _SUPPORT
|
||||
|
||||
Support = _reflection.GeneratedProtocolMessageType('Support', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SUPPORT,
|
||||
__module__ = 'support_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Support)
|
||||
))
|
||||
_sym_db.RegisterMessage(Support)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
+10
-11
@@ -7,7 +7,7 @@ def _create_url_regex():
|
||||
# see https://spec.lbry.com/ and test_url.py
|
||||
invalid_names_regex = \
|
||||
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
|
||||
r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+"
|
||||
r"\u0000-\u0008\u000b-\u000c\u000e-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]+"
|
||||
|
||||
def _named(name, regex):
|
||||
return "(?P<" + name + ">" + regex + ")"
|
||||
@@ -22,7 +22,8 @@ def _create_url_regex():
|
||||
return _group(
|
||||
_named(name+"_name", prefix + invalid_names_regex) +
|
||||
_oneof(
|
||||
_group('[:#]' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
|
||||
_group('#' + _named(name+"_claim_id", "[0-9a-f]{1,40}")),
|
||||
_group(':' + _named(name+"_sequence", '[1-9][0-9]*')),
|
||||
_group(r'\$' + _named(name+"_amount_order", '[1-9][0-9]*'))
|
||||
) + '?'
|
||||
)
|
||||
@@ -49,31 +50,28 @@ def normalize_name(name):
|
||||
class PathSegment(NamedTuple):
|
||||
name: str
|
||||
claim_id: str = None
|
||||
sequence: int = None
|
||||
amount_order: int = None
|
||||
|
||||
@property
|
||||
def normalized(self):
|
||||
return normalize_name(self.name)
|
||||
|
||||
@property
|
||||
def is_shortid(self):
|
||||
return self.claim_id is not None and len(self.claim_id) < 40
|
||||
|
||||
@property
|
||||
def is_fullid(self):
|
||||
return self.claim_id is not None and len(self.claim_id) == 40
|
||||
|
||||
def to_dict(self):
|
||||
q = {'name': self.name}
|
||||
if self.claim_id is not None:
|
||||
q['claim_id'] = self.claim_id
|
||||
if self.sequence is not None:
|
||||
q['sequence'] = self.sequence
|
||||
if self.amount_order is not None:
|
||||
q['amount_order'] = self.amount_order
|
||||
return q
|
||||
|
||||
def __str__(self):
|
||||
if self.claim_id is not None:
|
||||
return f"{self.name}:{self.claim_id}"
|
||||
return f"{self.name}#{self.claim_id}"
|
||||
elif self.sequence is not None:
|
||||
return f"{self.name}:{self.sequence}"
|
||||
elif self.amount_order is not None:
|
||||
return f"{self.name}${self.amount_order}"
|
||||
return self.name
|
||||
@@ -120,6 +118,7 @@ class URL(NamedTuple):
|
||||
segments[segment] = PathSegment(
|
||||
parts[f'{segment}_name'],
|
||||
parts[f'{segment}_claim_id'],
|
||||
parts[f'{segment}_sequence'],
|
||||
parts[f'{segment}_amount_order']
|
||||
)
|
||||
|
||||
|
||||
+12
-19
@@ -44,25 +44,18 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]:
|
||||
yield os.urandom(AES.block_size // 8)
|
||||
|
||||
|
||||
def read_bytes(file_path: str, offset: int, to_read: int):
|
||||
with open(file_path, 'rb') as f:
|
||||
f.seek(offset)
|
||||
return f.read(to_read)
|
||||
|
||||
|
||||
async def file_reader(file_path: str):
|
||||
def file_reader(file_path: str):
|
||||
length = int(os.stat(file_path).st_size)
|
||||
offset = 0
|
||||
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = await asyncio.get_event_loop().run_in_executor(
|
||||
None, read_bytes, file_path, offset, bytes_to_read
|
||||
)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
with open(file_path, 'rb') as stream_file:
|
||||
while offset < length:
|
||||
bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1)
|
||||
if not bytes_to_read:
|
||||
break
|
||||
blob_bytes = stream_file.read(bytes_to_read)
|
||||
yield blob_bytes
|
||||
offset += bytes_to_read
|
||||
|
||||
|
||||
def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'):
|
||||
@@ -183,11 +176,11 @@ class StreamDescriptor:
|
||||
raise InvalidStreamDescriptorError("Does not decode as valid JSON")
|
||||
if decoded['blobs'][-1]['length'] != 0:
|
||||
raise InvalidStreamDescriptorError("Does not end with a zero-length blob.")
|
||||
if any(blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]):
|
||||
if any([blob_info['length'] == 0 for blob_info in decoded['blobs'][:-1]]):
|
||||
raise InvalidStreamDescriptorError("Contains zero-length data blob")
|
||||
if 'blob_hash' in decoded['blobs'][-1]:
|
||||
raise InvalidStreamDescriptorError("Stream terminator blob should not have a hash")
|
||||
if any(i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])):
|
||||
if any([i != blob_info['blob_num'] for i, blob_info in enumerate(decoded['blobs'])]):
|
||||
raise InvalidStreamDescriptorError("Stream contains out of order or skipped blobs")
|
||||
descriptor = cls(
|
||||
loop, blob_dir,
|
||||
@@ -252,7 +245,7 @@ class StreamDescriptor:
|
||||
iv_generator = iv_generator or random_iv_generator()
|
||||
key = key or os.urandom(AES.block_size // 8)
|
||||
blob_num = -1
|
||||
async for blob_bytes in file_reader(file_path):
|
||||
for blob_bytes in file_reader(file_path):
|
||||
blob_num += 1
|
||||
blob_info = await BlobFile.create_from_unencrypted(
|
||||
loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback
|
||||
|
||||
@@ -40,7 +40,7 @@ class StreamDownloader:
|
||||
async def cached_read_blob(blob_info: 'BlobInfo') -> bytes:
|
||||
return await self.read_blob(blob_info, 2)
|
||||
|
||||
if self.blob_manager.decrypted_blob_lru_cache is not None:
|
||||
if self.blob_manager.decrypted_blob_lru_cache:
|
||||
cached_read_blob = lru_cache_concurrent(override_lru_cache=self.blob_manager.decrypted_blob_lru_cache)(
|
||||
cached_read_blob
|
||||
)
|
||||
@@ -51,15 +51,15 @@ class StreamDownloader:
|
||||
def _delayed_add_fixed_peers():
|
||||
self.added_fixed_peers = True
|
||||
self.peer_queue.put_nowait([
|
||||
make_kademlia_peer(None, address, None, tcp_port=port, allow_localhost=True)
|
||||
make_kademlia_peer(None, address, None, tcp_port=port + 1, allow_localhost=True)
|
||||
for address, port in addresses
|
||||
])
|
||||
|
||||
if not self.config.fixed_peers:
|
||||
if not self.config.reflector_servers:
|
||||
return
|
||||
addresses = [
|
||||
(await resolve_host(url, port, proto='tcp'), port)
|
||||
for url, port in self.config.fixed_peers
|
||||
(await resolve_host(url, port + 1, proto='tcp'), port)
|
||||
for url, port in self.config.reflector_servers
|
||||
]
|
||||
if 'dht' in self.config.components_to_skip or not self.node or not \
|
||||
len(self.node.protocol.routing_table.get_peers()) > 0:
|
||||
@@ -92,8 +92,8 @@ class StreamDownloader:
|
||||
|
||||
async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0):
|
||||
# set up peer accumulation
|
||||
self.node = node or self.node # fixme: this shouldnt be set here!
|
||||
if self.node:
|
||||
if node:
|
||||
self.node = node
|
||||
if self.accumulate_task and not self.accumulate_task.done():
|
||||
self.accumulate_task.cancel()
|
||||
_, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue)
|
||||
|
||||
+179
-89
@@ -3,8 +3,9 @@ import asyncio
|
||||
import time
|
||||
import typing
|
||||
import logging
|
||||
from typing import Optional
|
||||
import binascii
|
||||
from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable
|
||||
from lbry.utils import generate_id
|
||||
from lbry.error import DownloadSDTimeoutError
|
||||
from lbry.schema.mime_types import guess_media_type
|
||||
from lbry.stream.downloader import StreamDownloader
|
||||
@@ -12,7 +13,6 @@ from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name
|
||||
from lbry.stream.reflector.client import StreamReflectorClient
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
from lbry.blob import MAX_BLOB_SIZE
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
@@ -40,39 +40,76 @@ async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download
|
||||
return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name)
|
||||
|
||||
|
||||
class ManagedStream(ManagedDownloadSource):
|
||||
class ManagedStream:
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
SAVING_ID = 1
|
||||
STREAMING_ID = 2
|
||||
|
||||
__slots__ = [
|
||||
'loop',
|
||||
'config',
|
||||
'blob_manager',
|
||||
'sd_hash',
|
||||
'download_directory',
|
||||
'_file_name',
|
||||
'_added_on',
|
||||
'_status',
|
||||
'stream_claim_info',
|
||||
'download_id',
|
||||
'rowid',
|
||||
'content_fee',
|
||||
'purchase_receipt',
|
||||
'downloader',
|
||||
'analytics_manager',
|
||||
'fully_reflected',
|
||||
'file_output_task',
|
||||
'delayed_stop_task',
|
||||
'streaming_responses',
|
||||
'streaming',
|
||||
'_running',
|
||||
'saving',
|
||||
'finished_writing',
|
||||
'started_writing',
|
||||
'finished_write_attempt'
|
||||
]
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
sd_hash: str, download_directory: Optional[str] = None, file_name: Optional[str] = None,
|
||||
status: Optional[str] = ManagedDownloadSource.STATUS_STOPPED,
|
||||
claim: Optional[StoredContentClaim] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
descriptor: Optional[StreamDescriptor] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None):
|
||||
super().__init__(loop, config, blob_manager.storage, sd_hash, file_name, download_directory, status, claim,
|
||||
download_id, rowid, content_fee, analytics_manager, added_on)
|
||||
sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None,
|
||||
status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredContentClaim] = None,
|
||||
download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None,
|
||||
descriptor: typing.Optional[StreamDescriptor] = None,
|
||||
content_fee: typing.Optional['Transaction'] = None,
|
||||
analytics_manager: typing.Optional['AnalyticsManager'] = None,
|
||||
added_on: typing.Optional[int] = None):
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.sd_hash = sd_hash
|
||||
self.download_directory = download_directory
|
||||
self._file_name = file_name
|
||||
self._status = status
|
||||
self.stream_claim_info = claim
|
||||
self.download_id = download_id or binascii.hexlify(generate_id()).decode()
|
||||
self.rowid = rowid
|
||||
self.content_fee = content_fee
|
||||
self.purchase_receipt = None
|
||||
self._added_on = added_on
|
||||
self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor)
|
||||
self.analytics_manager = analytics_manager
|
||||
|
||||
self.reflector_progress = 0
|
||||
self.uploading_to_reflector = False
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.file_output_task: typing.Optional[asyncio.Task] = None
|
||||
self.delayed_stop_task: typing.Optional[asyncio.Task] = None
|
||||
self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = []
|
||||
self.fully_reflected = asyncio.Event(loop=self.loop)
|
||||
self.streaming = asyncio.Event(loop=self.loop)
|
||||
self._running = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def sd_hash(self) -> str:
|
||||
return self.identifier
|
||||
|
||||
@property
|
||||
def is_fully_reflected(self) -> bool:
|
||||
return self.fully_reflected.is_set()
|
||||
self.saving = asyncio.Event(loop=self.loop)
|
||||
self.finished_writing = asyncio.Event(loop=self.loop)
|
||||
self.started_writing = asyncio.Event(loop=self.loop)
|
||||
self.finished_write_attempt = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def descriptor(self) -> StreamDescriptor:
|
||||
@@ -83,9 +120,17 @@ class ManagedStream(ManagedDownloadSource):
|
||||
return self.descriptor.stream_hash
|
||||
|
||||
@property
|
||||
def file_name(self) -> Optional[str]:
|
||||
def file_name(self) -> typing.Optional[str]:
|
||||
return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None)
|
||||
|
||||
@property
|
||||
def added_on(self) -> typing.Optional[int]:
|
||||
return self._added_on
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return self._status
|
||||
|
||||
@property
|
||||
def written_bytes(self) -> int:
|
||||
return 0 if not self.output_file_exists else os.stat(self.full_path).st_size
|
||||
@@ -103,6 +148,55 @@ class ManagedStream(ManagedDownloadSource):
|
||||
self._status = status
|
||||
await self.blob_manager.storage.change_file_status(self.stream_hash, status)
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
return self.status == self.STATUS_FINISHED
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self.status == self.STATUS_RUNNING
|
||||
|
||||
@property
|
||||
def claim_id(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_id
|
||||
|
||||
@property
|
||||
def txid(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.txid
|
||||
|
||||
@property
|
||||
def nout(self) -> typing.Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.nout
|
||||
|
||||
@property
|
||||
def outpoint(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.outpoint
|
||||
|
||||
@property
|
||||
def claim_height(self) -> typing.Optional[int]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.height
|
||||
|
||||
@property
|
||||
def channel_claim_id(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id
|
||||
|
||||
@property
|
||||
def channel_name(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.channel_name
|
||||
|
||||
@property
|
||||
def claim_name(self) -> typing.Optional[str]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim_name
|
||||
|
||||
@property
|
||||
def metadata(self) -> typing.Optional[typing.Dict]:
|
||||
return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict()
|
||||
|
||||
@property
|
||||
def metadata_protobuf(self) -> bytes:
|
||||
if self.stream_claim_info:
|
||||
return binascii.hexlify(self.stream_claim_info.claim.to_bytes())
|
||||
|
||||
@property
|
||||
def blobs_completed(self) -> int:
|
||||
return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0
|
||||
@@ -116,34 +210,39 @@ class ManagedStream(ManagedDownloadSource):
|
||||
def blobs_remaining(self) -> int:
|
||||
return self.blobs_in_stream - self.blobs_completed
|
||||
|
||||
@property
|
||||
def full_path(self) -> typing.Optional[str]:
|
||||
return os.path.join(self.download_directory, os.path.basename(self.file_name)) \
|
||||
if self.file_name and self.download_directory else None
|
||||
|
||||
@property
|
||||
def output_file_exists(self):
|
||||
return os.path.isfile(self.full_path) if self.full_path else False
|
||||
|
||||
@property
|
||||
def mime_type(self):
|
||||
return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0]
|
||||
|
||||
@property
|
||||
def download_path(self):
|
||||
return f"{self.download_directory}/{self._file_name}" if self.download_directory and self._file_name else None
|
||||
@classmethod
|
||||
async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
file_path: str, key: typing.Optional[bytes] = None,
|
||||
iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream':
|
||||
"""
|
||||
Generate a stream from a file and save it to the db
|
||||
"""
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
blob_completed_callback=blob_manager.blob_completed
|
||||
)
|
||||
await blob_manager.storage.store_stream(
|
||||
blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
)
|
||||
row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||
os.path.dirname(file_path), 0)
|
||||
return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
|
||||
|
||||
# @classmethod
|
||||
# async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config',
|
||||
# file_path: str, key: Optional[bytes] = None,
|
||||
# iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource':
|
||||
# """
|
||||
# Generate a stream from a file and save it to the db
|
||||
# """
|
||||
# descriptor = await StreamDescriptor.create_stream(
|
||||
# loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
# blob_completed_callback=blob_manager.blob_completed
|
||||
# )
|
||||
# await blob_manager.storage.store_stream(
|
||||
# blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
# )
|
||||
# row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path),
|
||||
# os.path.dirname(file_path), 0)
|
||||
# return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
# os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor)
|
||||
|
||||
async def start(self, timeout: Optional[float] = None,
|
||||
async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None,
|
||||
save_now: bool = False):
|
||||
timeout = timeout or self.config.download_timeout
|
||||
if self._running.is_set():
|
||||
@@ -151,7 +250,7 @@ class ManagedStream(ManagedDownloadSource):
|
||||
log.info("start downloader for stream (sd hash: %s)", self.sd_hash)
|
||||
self._running.set()
|
||||
try:
|
||||
await asyncio.wait_for(self.downloader.start(), timeout, loop=self.loop)
|
||||
await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop)
|
||||
except asyncio.TimeoutError:
|
||||
self._running.clear()
|
||||
raise DownloadSDTimeoutError(self.sd_hash)
|
||||
@@ -161,11 +260,6 @@ class ManagedStream(ManagedDownloadSource):
|
||||
self.delayed_stop_task = self.loop.create_task(self._delayed_stop())
|
||||
if not await self.blob_manager.storage.file_exists(self.sd_hash):
|
||||
if save_now:
|
||||
if not self._file_name:
|
||||
self._file_name = await get_next_available_file_name(
|
||||
self.loop, self.download_directory,
|
||||
self._file_name or sanitize_file_name(self.descriptor.suggested_file_name)
|
||||
)
|
||||
file_name, download_dir = self._file_name, self.download_directory
|
||||
else:
|
||||
file_name, download_dir = None, None
|
||||
@@ -185,7 +279,7 @@ class ManagedStream(ManagedDownloadSource):
|
||||
if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING:
|
||||
await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED)
|
||||
|
||||
async def _aiter_read_stream(self, start_blob_num: Optional[int] = 0, connection_id: int = 0)\
|
||||
async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\
|
||||
-> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]:
|
||||
if start_blob_num >= len(self.descriptor.blobs[:-1]):
|
||||
raise IndexError(start_blob_num)
|
||||
@@ -197,13 +291,13 @@ class ManagedStream(ManagedDownloadSource):
|
||||
decrypted = await self.downloader.read_blob(blob_info, connection_id)
|
||||
yield (blob_info, decrypted)
|
||||
|
||||
async def stream_file(self, request: Request) -> StreamResponse:
|
||||
async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse:
|
||||
log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id,
|
||||
self.sd_hash[:6])
|
||||
headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers(
|
||||
request.headers.get('range', 'bytes=0-')
|
||||
)
|
||||
await self.start()
|
||||
await self.start(node)
|
||||
response = StreamResponse(
|
||||
status=206,
|
||||
headers=headers
|
||||
@@ -241,10 +335,9 @@ class ManagedStream(ManagedDownloadSource):
|
||||
self.streaming.clear()
|
||||
|
||||
@staticmethod
|
||||
def _write_decrypted_blob(output_path: str, data: bytes):
|
||||
with open(output_path, 'ab') as handle:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
def _write_decrypted_blob(handle: typing.IO, data: bytes):
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
|
||||
async def _save_file(self, output_path: str):
|
||||
log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6],
|
||||
@@ -254,12 +347,12 @@ class ManagedStream(ManagedDownloadSource):
|
||||
self.finished_writing.clear()
|
||||
self.started_writing.clear()
|
||||
try:
|
||||
open(output_path, 'wb').close()
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
with open(output_path, 'wb') as file_write_handle:
|
||||
async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID):
|
||||
log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1)
|
||||
await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted)
|
||||
if not self.started_writing.is_set():
|
||||
self.started_writing.set()
|
||||
await self.update_status(ManagedStream.STATUS_FINISHED)
|
||||
if self.analytics_manager:
|
||||
self.loop.create_task(self.analytics_manager.send_download_finished(
|
||||
@@ -289,8 +382,9 @@ class ManagedStream(ManagedDownloadSource):
|
||||
self.saving.clear()
|
||||
self.finished_write_attempt.set()
|
||||
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
await self.start()
|
||||
async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None,
|
||||
node: typing.Optional['Node'] = None):
|
||||
await self.start(node)
|
||||
if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task
|
||||
self.file_output_task.cancel()
|
||||
self.download_directory = download_directory or self.download_directory or self.config.download_dir
|
||||
@@ -332,13 +426,12 @@ class ManagedStream(ManagedDownloadSource):
|
||||
sent = []
|
||||
protocol = StreamReflectorClient(self.blob_manager, self.descriptor)
|
||||
try:
|
||||
self.uploading_to_reflector = True
|
||||
await self.loop.create_connection(lambda: protocol, host, port)
|
||||
await protocol.send_handshake()
|
||||
sent_sd, needed = await protocol.send_descriptor()
|
||||
if sent_sd: # reflector needed the sd blob
|
||||
if sent_sd:
|
||||
sent.append(self.sd_hash)
|
||||
if not sent_sd and not needed: # reflector already has the stream
|
||||
if not sent_sd and not needed:
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
@@ -348,33 +441,30 @@ class ManagedStream(ManagedDownloadSource):
|
||||
]
|
||||
log.info("we have %i/%i needed blobs needed by reflector for lbry://%s#%s", len(we_have), len(needed),
|
||||
self.claim_name, self.claim_id)
|
||||
for i, blob_hash in enumerate(we_have):
|
||||
for blob_hash in we_have:
|
||||
await protocol.send_blob(blob_hash)
|
||||
sent.append(blob_hash)
|
||||
self.reflector_progress = int((i + 1) / len(we_have) * 100)
|
||||
except (asyncio.TimeoutError, ValueError):
|
||||
return sent
|
||||
except ConnectionError:
|
||||
return sent
|
||||
except (OSError, Exception) as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
log.warning("stopped uploading %s#%s to reflector", self.claim_name, self.claim_id)
|
||||
elif isinstance(err, OSError):
|
||||
log.warning(
|
||||
"stopped uploading %s#%s to reflector because blobs were deleted or moved", self.claim_name,
|
||||
self.claim_id
|
||||
)
|
||||
else:
|
||||
log.exception("unexpected error reflecting %s#%s", self.claim_name, self.claim_id)
|
||||
except ConnectionRefusedError:
|
||||
return sent
|
||||
finally:
|
||||
if protocol.transport:
|
||||
protocol.transport.close()
|
||||
self.uploading_to_reflector = False
|
||||
|
||||
if not self.fully_reflected.is_set():
|
||||
self.fully_reflected.set()
|
||||
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
|
||||
return sent
|
||||
|
||||
async def update_content_claim(self, claim_info: Optional[typing.Dict] = None):
|
||||
def set_claim(self, claim_info: typing.Dict, claim: 'Claim'):
|
||||
self.stream_claim_info = StoredContentClaim(
|
||||
f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'],
|
||||
claim_info['name'], claim_info['amount'], claim_info['height'],
|
||||
binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'],
|
||||
claim_info['claim_sequence'], claim_info.get('channel_name')
|
||||
)
|
||||
|
||||
async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None):
|
||||
if not claim_info:
|
||||
claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash)
|
||||
self.set_claim(claim_info, claim_info['value'])
|
||||
|
||||
@@ -35,8 +35,6 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
def connection_lost(self, exc: typing.Optional[Exception]):
|
||||
self.transport = None
|
||||
self.connected.clear()
|
||||
if self.pending_request:
|
||||
self.pending_request.cancel()
|
||||
if self.reflected_blobs:
|
||||
log.info("Finished sending reflector %i blobs", len(self.reflected_blobs))
|
||||
|
||||
@@ -58,18 +56,12 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
self.response_buff = b''
|
||||
return
|
||||
|
||||
async def send_request(self, request_dict: typing.Dict, timeout: int = 180):
|
||||
msg = json.dumps(request_dict, sort_keys=True)
|
||||
async def send_request(self, request_dict: typing.Dict):
|
||||
msg = json.dumps(request_dict)
|
||||
self.transport.write(msg.encode())
|
||||
try:
|
||||
self.transport.write(msg.encode())
|
||||
self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout))
|
||||
self.pending_request = self.loop.create_task(self.response_queue.get())
|
||||
return await self.pending_request
|
||||
except (AttributeError, asyncio.CancelledError) as err:
|
||||
# attribute error happens when we transport.write after disconnect
|
||||
# cancelled error happens when the pending_request task is cancelled by a disconnect
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
raise err if isinstance(err, asyncio.CancelledError) else asyncio.CancelledError()
|
||||
finally:
|
||||
self.pending_request = None
|
||||
|
||||
@@ -94,16 +86,8 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
needed = response.get('needed_blobs', [])
|
||||
sent_sd = False
|
||||
if response['send_sd_blob']:
|
||||
try:
|
||||
sent = await sd_blob.sendfile(self)
|
||||
if sent == -1:
|
||||
log.warning("failed to send sd blob")
|
||||
raise asyncio.CancelledError()
|
||||
received = await asyncio.wait_for(self.response_queue.get(), 30)
|
||||
except asyncio.CancelledError as err:
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
raise err
|
||||
await sd_blob.sendfile(self)
|
||||
received = await self.response_queue.get()
|
||||
if received.get('received_sd_blob'):
|
||||
sent_sd = True
|
||||
if not needed:
|
||||
@@ -126,16 +110,8 @@ class StreamReflectorClient(asyncio.Protocol):
|
||||
if 'send_blob' not in response:
|
||||
raise ValueError("I don't know whether to send the blob or not!")
|
||||
if response['send_blob']:
|
||||
try:
|
||||
sent = await blob.sendfile(self)
|
||||
if sent == -1:
|
||||
log.warning("failed to send blob")
|
||||
raise asyncio.CancelledError()
|
||||
received = await asyncio.wait_for(self.response_queue.get(), 30)
|
||||
except asyncio.CancelledError as err:
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
raise err
|
||||
await blob.sendfile(self)
|
||||
received = await self.response_queue.get()
|
||||
if received.get('received_blob'):
|
||||
self.reflected_blobs.append(blob.blob_hash)
|
||||
log.info("Sent reflector blob %s", blob.blob_hash[:8])
|
||||
|
||||
@@ -15,9 +15,7 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReflectorServerProtocol(asyncio.Protocol):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||
not_incoming_event: asyncio.Event = None, partial_event: asyncio.Event = None):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: asyncio.Task = None
|
||||
@@ -29,26 +27,11 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.descriptor: typing.Optional['StreamDescriptor'] = None
|
||||
self.sd_blob: typing.Optional['BlobFile'] = None
|
||||
self.received = []
|
||||
self.incoming = incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.not_incoming = not_incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.stop_event = stop_event or asyncio.Event(loop=self.loop)
|
||||
self.incoming = asyncio.Event(loop=self.loop)
|
||||
self.chunk_size = response_chunk_size
|
||||
self.wait_for_stop_task: typing.Optional[asyncio.Task] = None
|
||||
self.partial_event = partial_event
|
||||
|
||||
async def wait_for_stop(self):
|
||||
await self.stop_event.wait()
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
|
||||
def connection_made(self, transport):
|
||||
self.transport = transport
|
||||
self.wait_for_stop_task = self.loop.create_task(self.wait_for_stop())
|
||||
|
||||
def connection_lost(self, exc):
|
||||
if self.wait_for_stop_task:
|
||||
self.wait_for_stop_task.cancel()
|
||||
self.wait_for_stop_task = None
|
||||
|
||||
def data_received(self, data: bytes):
|
||||
if self.incoming.is_set():
|
||||
@@ -90,7 +73,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.sd_blob = self.blob_manager.get_blob(request['sd_blob_hash'], request['sd_blob_size'])
|
||||
if not self.sd_blob.get_is_verified():
|
||||
self.writer = self.sd_blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||
self.not_incoming.clear()
|
||||
self.incoming.set()
|
||||
self.send_response({"send_sd_blob": True})
|
||||
try:
|
||||
@@ -104,7 +86,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.transport.close()
|
||||
finally:
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
else:
|
||||
@@ -112,18 +93,13 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
self.loop, self.blob_manager.blob_dir, self.sd_blob
|
||||
)
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
if self.writer:
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
|
||||
needs = [blob.blob_hash
|
||||
for blob in self.descriptor.blobs[:-1]
|
||||
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()]
|
||||
if needs and not self.partial_event.is_set():
|
||||
needs = needs[:3]
|
||||
self.partial_event.set()
|
||||
self.send_response({"send_sd_blob": False, 'needed_blobs': needs})
|
||||
self.send_response({"send_sd_blob": False, 'needed': [
|
||||
blob.blob_hash for blob in self.descriptor.blobs[:-1]
|
||||
if not self.blob_manager.get_blob(blob.blob_hash).get_is_verified()
|
||||
]})
|
||||
return
|
||||
return
|
||||
elif self.descriptor:
|
||||
@@ -136,7 +112,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
blob = self.blob_manager.get_blob(request['blob_hash'], request['blob_size'])
|
||||
if not blob.get_is_verified():
|
||||
self.writer = blob.get_blob_writer(self.transport.get_extra_info('peername'))
|
||||
self.not_incoming.clear()
|
||||
self.incoming.set()
|
||||
self.send_response({"send_blob": True})
|
||||
try:
|
||||
@@ -145,7 +120,6 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
except asyncio.TimeoutError:
|
||||
self.send_response({"received_blob": False})
|
||||
self.incoming.clear()
|
||||
self.not_incoming.set()
|
||||
self.writer.close_handle()
|
||||
self.writer = None
|
||||
else:
|
||||
@@ -156,39 +130,26 @@ class ReflectorServerProtocol(asyncio.Protocol):
|
||||
|
||||
|
||||
class ReflectorServer:
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000,
|
||||
stop_event: asyncio.Event = None, incoming_event: asyncio.Event = None,
|
||||
not_incoming_event: asyncio.Event = None, partial_needs=False):
|
||||
def __init__(self, blob_manager: 'BlobManager', response_chunk_size: int = 10000):
|
||||
self.loop = asyncio.get_event_loop()
|
||||
self.blob_manager = blob_manager
|
||||
self.server_task: typing.Optional[asyncio.Task] = None
|
||||
self.started_listening = asyncio.Event(loop=self.loop)
|
||||
self.stopped_listening = asyncio.Event(loop=self.loop)
|
||||
self.incoming_event = incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.not_incoming_event = not_incoming_event or asyncio.Event(loop=self.loop)
|
||||
self.response_chunk_size = response_chunk_size
|
||||
self.stop_event = stop_event
|
||||
self.partial_needs = partial_needs # for testing cases where it doesn't know what it wants
|
||||
|
||||
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
|
||||
if self.server_task is not None:
|
||||
raise Exception("already running")
|
||||
|
||||
async def _start_server():
|
||||
partial_event = asyncio.Event()
|
||||
if not self.partial_needs:
|
||||
partial_event.set()
|
||||
server = await self.loop.create_server(lambda: ReflectorServerProtocol(
|
||||
self.blob_manager, self.response_chunk_size, self.stop_event, self.incoming_event,
|
||||
self.not_incoming_event, partial_event), interface, port)
|
||||
server = await self.loop.create_server(
|
||||
lambda: ReflectorServerProtocol(self.blob_manager, self.response_chunk_size),
|
||||
interface, port
|
||||
)
|
||||
self.started_listening.set()
|
||||
self.stopped_listening.clear()
|
||||
log.info("Reflector server listening on TCP %s:%i", interface, port)
|
||||
try:
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
finally:
|
||||
self.stopped_listening.set()
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
|
||||
self.server_task = self.loop.create_task(_start_server())
|
||||
|
||||
|
||||
+334
-127
@@ -6,67 +6,83 @@ import random
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.error import InvalidStreamDescriptorError
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError
|
||||
from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError
|
||||
from lbry.utils import cache_concurrent
|
||||
from lbry.stream.descriptor import StreamDescriptor
|
||||
from lbry.stream.managed_stream import ManagedStream
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.schema.url import URL
|
||||
from lbry.wallet.dewies import dewies_to_lbc
|
||||
from lbry.wallet import Output
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.conf import Config
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
from lbry.dht.node import Node
|
||||
from lbry.wallet.wallet import WalletManager
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
FILTER_FIELDS = [
|
||||
'rowid',
|
||||
'status',
|
||||
'file_name',
|
||||
'added_on',
|
||||
'sd_hash',
|
||||
'stream_hash',
|
||||
'claim_name',
|
||||
'claim_height',
|
||||
'claim_id',
|
||||
'outpoint',
|
||||
'txid',
|
||||
'nout',
|
||||
'channel_claim_id',
|
||||
'channel_name',
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream'
|
||||
]
|
||||
|
||||
def path_or_none(encoded_path) -> Optional[str]:
|
||||
if not encoded_path:
|
||||
COMPARISON_OPERATORS = {
|
||||
'eq': lambda a, b: a == b,
|
||||
'ne': lambda a, b: a != b,
|
||||
'g': lambda a, b: a > b,
|
||||
'l': lambda a, b: a < b,
|
||||
'ge': lambda a, b: a >= b,
|
||||
'le': lambda a, b: a <= b,
|
||||
}
|
||||
|
||||
|
||||
def path_or_none(path) -> Optional[str]:
|
||||
if not path:
|
||||
return
|
||||
return binascii.unhexlify(encoded_path).decode()
|
||||
return binascii.unhexlify(path).decode()
|
||||
|
||||
|
||||
class StreamManager(SourceManager):
|
||||
_sources: typing.Dict[str, ManagedStream]
|
||||
|
||||
filter_fields = SourceManager.filter_fields
|
||||
filter_fields.update({
|
||||
'sd_hash',
|
||||
'stream_hash',
|
||||
'full_status', # TODO: remove
|
||||
'blobs_remaining',
|
||||
'blobs_in_stream',
|
||||
'uploading_to_reflector',
|
||||
'is_fully_reflected'
|
||||
})
|
||||
|
||||
class StreamManager:
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager',
|
||||
wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'],
|
||||
analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
super().__init__(loop, config, storage, analytics_manager)
|
||||
self.loop = loop
|
||||
self.config = config
|
||||
self.blob_manager = blob_manager
|
||||
self.wallet_manager = wallet_manager
|
||||
self.storage = storage
|
||||
self.node = node
|
||||
self.analytics_manager = analytics_manager
|
||||
self.streams: typing.Dict[str, ManagedStream] = {}
|
||||
self.resume_saving_task: Optional[asyncio.Task] = None
|
||||
self.re_reflect_task: Optional[asyncio.Task] = None
|
||||
self.update_stream_finished_futs: typing.List[asyncio.Future] = []
|
||||
self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {}
|
||||
self.running_reflector_uploads: typing.List[asyncio.Task] = []
|
||||
self.started = asyncio.Event(loop=self.loop)
|
||||
|
||||
@property
|
||||
def streams(self):
|
||||
return self._sources
|
||||
|
||||
def add(self, source: ManagedStream):
|
||||
super().add(source)
|
||||
self.storage.content_claim_callbacks[source.stream_hash] = lambda: self._update_content_claim(source)
|
||||
|
||||
async def _update_content_claim(self, stream: ManagedStream):
|
||||
claim_info = await self.storage.get_content_claim(stream.stream_hash)
|
||||
self._sources.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
|
||||
self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value'])
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
to_restore = []
|
||||
@@ -97,10 +113,10 @@ class StreamManager(SourceManager):
|
||||
# if self.blob_manager._save_blobs:
|
||||
# log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos))
|
||||
|
||||
async def _load_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int], fully_reflected: Optional[bool]):
|
||||
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int]):
|
||||
try:
|
||||
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
|
||||
except InvalidStreamDescriptorError as err:
|
||||
@@ -111,11 +127,10 @@ class StreamManager(SourceManager):
|
||||
claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor,
|
||||
analytics_manager=self.analytics_manager, added_on=added_on
|
||||
)
|
||||
if fully_reflected:
|
||||
stream.fully_reflected.set()
|
||||
self.add(stream)
|
||||
self.streams[sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
|
||||
async def initialize_from_database(self):
|
||||
async def load_and_resume_streams_from_database(self):
|
||||
to_recover = []
|
||||
to_start = []
|
||||
|
||||
@@ -129,6 +144,7 @@ class StreamManager(SourceManager):
|
||||
to_recover.append(file_info)
|
||||
to_start.append(file_info)
|
||||
if to_recover:
|
||||
log.info("Recover %i files", len(to_recover))
|
||||
await self.recover_streams(to_recover)
|
||||
|
||||
log.info("Initializing %i files", len(to_start))
|
||||
@@ -139,135 +155,326 @@ class StreamManager(SourceManager):
|
||||
download_directory = path_or_none(file_info['download_directory'])
|
||||
if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running':
|
||||
to_resume_saving.append((file_name, download_directory, file_info['sd_hash']))
|
||||
add_stream_tasks.append(self.loop.create_task(self._load_stream(
|
||||
add_stream_tasks.append(self.loop.create_task(self.add_stream(
|
||||
file_info['rowid'], file_info['sd_hash'], file_name,
|
||||
download_directory, file_info['status'],
|
||||
file_info['claim'], file_info['content_fee'],
|
||||
file_info['added_on'], file_info['fully_reflected']
|
||||
file_info['added_on']
|
||||
)))
|
||||
if add_stream_tasks:
|
||||
await asyncio.gather(*add_stream_tasks, loop=self.loop)
|
||||
log.info("Started stream manager with %i files", len(self._sources))
|
||||
log.info("Started stream manager with %i files", len(self.streams))
|
||||
if not self.node:
|
||||
log.info("no DHT node given, resuming downloads trusting that we can contact reflector")
|
||||
if to_resume_saving:
|
||||
log.info("Resuming saving %i files", len(to_resume_saving))
|
||||
self.resume_saving_task = asyncio.ensure_future(asyncio.gather(
|
||||
*(self._sources[sd_hash].save_file(file_name, download_directory)
|
||||
for (file_name, download_directory, sd_hash) in to_resume_saving),
|
||||
loop=self.loop
|
||||
))
|
||||
self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving))
|
||||
|
||||
async def resume(self, to_resume_saving):
|
||||
log.info("Resuming saving %i files", len(to_resume_saving))
|
||||
await asyncio.gather(
|
||||
*(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node)
|
||||
for (file_name, download_directory, sd_hash) in to_resume_saving),
|
||||
loop=self.loop
|
||||
)
|
||||
|
||||
async def reflect_streams(self):
|
||||
try:
|
||||
return await self._reflect_streams()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
log.exception("reflector task encountered an unexpected error!")
|
||||
|
||||
async def _reflect_streams(self):
|
||||
# todo: those debug statements are temporary for #2987 - remove them if its closed
|
||||
while True:
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
log.debug("collecting streams to reflect")
|
||||
sd_hashes = await self.storage.get_streams_to_re_reflect()
|
||||
sd_hashes = [sd for sd in sd_hashes if sd in self._sources]
|
||||
sd_hashes = [sd for sd in sd_hashes if sd in self.streams]
|
||||
batch = []
|
||||
while sd_hashes:
|
||||
stream = self.streams[sd_hashes.pop()]
|
||||
if self.blob_manager.is_blob_verified(stream.sd_hash) and stream.blobs_completed and \
|
||||
stream.sd_hash not in self.running_reflector_uploads and not \
|
||||
stream.fully_reflected.is_set():
|
||||
batch.append(self.reflect_stream(stream))
|
||||
if self.blob_manager.is_blob_verified(stream.sd_hash) and stream.blobs_completed:
|
||||
if not stream.fully_reflected.is_set():
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
batch.append(stream.upload_to_reflector(host, port))
|
||||
if len(batch) >= self.config.concurrent_reflector_uploads:
|
||||
log.debug("waiting for batch of %s reflecting streams", len(batch))
|
||||
await asyncio.gather(*batch, loop=self.loop)
|
||||
log.debug("done processing %s streams", len(batch))
|
||||
batch = []
|
||||
if batch:
|
||||
log.debug("waiting for batch of %s reflecting streams", len(batch))
|
||||
await asyncio.gather(*batch, loop=self.loop)
|
||||
log.debug("done processing %s streams", len(batch))
|
||||
await asyncio.sleep(300, loop=self.loop)
|
||||
|
||||
async def start(self):
|
||||
await super().start()
|
||||
await self.load_and_resume_streams_from_database()
|
||||
self.re_reflect_task = self.loop.create_task(self.reflect_streams())
|
||||
self.started.set()
|
||||
|
||||
def stop(self):
|
||||
super().stop()
|
||||
if self.resume_saving_task and not self.resume_saving_task.done():
|
||||
self.resume_saving_task.cancel()
|
||||
if self.re_reflect_task and not self.re_reflect_task.done():
|
||||
self.re_reflect_task.cancel()
|
||||
while self.streams:
|
||||
_, stream = self.streams.popitem()
|
||||
stream.stop_tasks()
|
||||
while self.update_stream_finished_futs:
|
||||
self.update_stream_finished_futs.pop().cancel()
|
||||
while self.running_reflector_uploads:
|
||||
_, t = self.running_reflector_uploads.popitem()
|
||||
t.cancel()
|
||||
self.running_reflector_uploads.pop().cancel()
|
||||
self.started.clear()
|
||||
log.info("finished stopping the stream manager")
|
||||
|
||||
def reflect_stream(self, stream: ManagedStream, server: Optional[str] = None,
|
||||
port: Optional[int] = None) -> asyncio.Task:
|
||||
if not server or not port:
|
||||
server, port = random.choice(self.config.reflector_servers)
|
||||
if stream.sd_hash in self.running_reflector_uploads:
|
||||
return self.running_reflector_uploads[stream.sd_hash]
|
||||
task = self.loop.create_task(self._retriable_reflect_stream(stream, server, port))
|
||||
self.running_reflector_uploads[stream.sd_hash] = task
|
||||
task.add_done_callback(
|
||||
lambda _: None if stream.sd_hash not in self.running_reflector_uploads else
|
||||
self.running_reflector_uploads.pop(stream.sd_hash)
|
||||
)
|
||||
return task
|
||||
|
||||
async def _retriable_reflect_stream(self, stream, host, port):
|
||||
sent = await stream.upload_to_reflector(host, port)
|
||||
while not stream.is_fully_reflected and stream.reflector_progress > 0 and len(sent) > 0:
|
||||
stream.reflector_progress = 0
|
||||
sent = await stream.upload_to_reflector(host, port)
|
||||
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
descriptor = await StreamDescriptor.create_stream(
|
||||
self.loop, self.blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator,
|
||||
blob_completed_callback=self.blob_manager.blob_completed
|
||||
)
|
||||
await self.storage.store_stream(
|
||||
self.blob_manager.get_blob(descriptor.sd_hash), descriptor
|
||||
)
|
||||
row_id = await self.storage.save_published_file(
|
||||
descriptor.stream_hash, os.path.basename(file_path), os.path.dirname(file_path), 0
|
||||
)
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, self.blob_manager, descriptor.sd_hash, os.path.dirname(file_path),
|
||||
os.path.basename(file_path), status=ManagedDownloadSource.STATUS_FINISHED,
|
||||
rowid=row_id, descriptor=descriptor
|
||||
)
|
||||
async def create_stream(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream:
|
||||
stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator)
|
||||
self.streams[stream.sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
if self.config.reflect_streams and self.config.reflector_servers:
|
||||
self.reflect_stream(stream)
|
||||
host, port = random.choice(self.config.reflector_servers)
|
||||
task = self.loop.create_task(stream.upload_to_reflector(host, port))
|
||||
self.running_reflector_uploads.append(task)
|
||||
task.add_done_callback(
|
||||
lambda _: None
|
||||
if task not in self.running_reflector_uploads else self.running_reflector_uploads.remove(task)
|
||||
)
|
||||
return stream
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
if not isinstance(source, ManagedStream):
|
||||
return
|
||||
if source.identifier in self.running_reflector_uploads:
|
||||
self.running_reflector_uploads[source.identifier].cancel()
|
||||
source.stop_tasks()
|
||||
if source.identifier in self.streams:
|
||||
del self.streams[source.identifier]
|
||||
blob_hashes = [source.identifier] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
|
||||
async def delete_stream(self, stream: ManagedStream, delete_file: Optional[bool] = False):
|
||||
stream.stop_tasks()
|
||||
if stream.sd_hash in self.streams:
|
||||
del self.streams[stream.sd_hash]
|
||||
blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]]
|
||||
await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||
await self.storage.delete_stream(source.descriptor)
|
||||
if delete_file and source.output_file_exists:
|
||||
os.remove(source.full_path)
|
||||
await self.storage.delete_stream(stream.descriptor)
|
||||
if delete_file and stream.output_file_exists:
|
||||
os.remove(stream.full_path)
|
||||
|
||||
def get_stream_by_stream_hash(self, stream_hash: str) -> Optional[ManagedStream]:
|
||||
streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values()))
|
||||
if streams:
|
||||
return streams[0]
|
||||
|
||||
def get_filtered_streams(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False,
|
||||
comparison: Optional[str] = None,
|
||||
**search_by) -> typing.List[ManagedStream]:
|
||||
"""
|
||||
Get a list of filtered and sorted ManagedStream objects
|
||||
|
||||
:param sort_by: field to sort by
|
||||
:param reverse: reverse sorting
|
||||
:param comparison: comparison operator used for filtering
|
||||
:param search_by: fields and values to filter by
|
||||
"""
|
||||
if sort_by and sort_by not in FILTER_FIELDS:
|
||||
raise ValueError(f"'{sort_by}' is not a valid field to sort by")
|
||||
if comparison and comparison not in COMPARISON_OPERATORS:
|
||||
raise ValueError(f"'{comparison}' is not a valid comparison")
|
||||
if 'full_status' in search_by:
|
||||
del search_by['full_status']
|
||||
for search in search_by:
|
||||
if search not in FILTER_FIELDS:
|
||||
raise ValueError(f"'{search}' is not a valid search operation")
|
||||
if search_by:
|
||||
comparison = comparison or 'eq'
|
||||
streams = []
|
||||
for stream in self.streams.values():
|
||||
for search, val in search_by.items():
|
||||
if COMPARISON_OPERATORS[comparison](getattr(stream, search), val):
|
||||
streams.append(stream)
|
||||
break
|
||||
else:
|
||||
streams = list(self.streams.values())
|
||||
if sort_by:
|
||||
streams.sort(key=lambda s: getattr(s, sort_by))
|
||||
if reverse:
|
||||
streams.reverse()
|
||||
return streams
|
||||
|
||||
async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim
|
||||
) -> typing.Tuple[Optional[ManagedStream], Optional[ManagedStream]]:
|
||||
existing = self.get_filtered_streams(outpoint=outpoint)
|
||||
if existing:
|
||||
return existing[0], None
|
||||
existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash)
|
||||
if existing and existing[0].claim_id != claim_id:
|
||||
raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}")
|
||||
if existing:
|
||||
log.info("claim contains a metadata only update to a stream we have")
|
||||
await self.storage.save_content_claim(
|
||||
existing[0].stream_hash, outpoint
|
||||
)
|
||||
await self._update_content_claim(existing[0])
|
||||
return existing[0], None
|
||||
else:
|
||||
existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id)
|
||||
if existing_for_claim_id:
|
||||
log.info("claim contains an update to a stream we have, downloading it")
|
||||
return None, existing_for_claim_id[0]
|
||||
return None, None
|
||||
|
||||
@staticmethod
|
||||
def _convert_to_old_resolve_output(wallet_manager, resolves):
|
||||
result = {}
|
||||
for url, txo in resolves.items():
|
||||
if isinstance(txo, Output):
|
||||
tx_height = txo.tx_ref.height
|
||||
best_height = wallet_manager.ledger.headers.height
|
||||
result[url] = {
|
||||
'name': txo.claim_name,
|
||||
'value': txo.claim,
|
||||
'protobuf': binascii.hexlify(txo.claim.to_bytes()),
|
||||
'claim_id': txo.claim_id,
|
||||
'txid': txo.tx_ref.id,
|
||||
'nout': txo.position,
|
||||
'amount': dewies_to_lbc(txo.amount),
|
||||
'effective_amount': txo.meta.get('effective_amount', 0),
|
||||
'height': tx_height,
|
||||
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
|
||||
'claim_sequence': -1,
|
||||
'address': txo.get_address(wallet_manager.ledger),
|
||||
'valid_at_height': txo.meta.get('activation_height', None),
|
||||
'timestamp': wallet_manager.ledger.headers[tx_height]['timestamp'],
|
||||
'supports': []
|
||||
}
|
||||
else:
|
||||
result[url] = txo
|
||||
return result
|
||||
|
||||
@cache_concurrent
|
||||
async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager',
|
||||
timeout: Optional[float] = None,
|
||||
file_name: Optional[str] = None,
|
||||
download_directory: Optional[str] = None,
|
||||
save_file: Optional[bool] = None,
|
||||
resolve_timeout: float = 3.0,
|
||||
wallet: Optional['Wallet'] = None) -> ManagedStream:
|
||||
manager = self.wallet_manager
|
||||
wallet = wallet or manager.default_wallet
|
||||
timeout = timeout or self.config.download_timeout
|
||||
start_time = self.loop.time()
|
||||
resolved_time = None
|
||||
stream = None
|
||||
txo: Optional[Output] = None
|
||||
error = None
|
||||
outpoint = None
|
||||
if save_file is None:
|
||||
save_file = self.config.save_files
|
||||
if file_name and not save_file:
|
||||
save_file = True
|
||||
if save_file:
|
||||
download_directory = download_directory or self.config.download_dir
|
||||
else:
|
||||
download_directory = None
|
||||
|
||||
payment = None
|
||||
try:
|
||||
# resolve the claim
|
||||
if not URL.parse(uri).has_stream:
|
||||
raise ResolveError("cannot download a channel claim, specify a /path")
|
||||
try:
|
||||
response = await asyncio.wait_for(
|
||||
manager.ledger.resolve(wallet.accounts, [uri]),
|
||||
resolve_timeout
|
||||
)
|
||||
resolved_result = self._convert_to_old_resolve_output(manager, response)
|
||||
except asyncio.TimeoutError:
|
||||
raise ResolveTimeoutError(uri)
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8
|
||||
raise
|
||||
log.exception("Unexpected error resolving stream:")
|
||||
raise ResolveError(f"Unexpected error resolving stream: {str(err)}")
|
||||
await self.storage.save_claims_for_resolve([
|
||||
value for value in resolved_result.values() if 'error' not in value
|
||||
])
|
||||
resolved = resolved_result.get(uri, {})
|
||||
resolved = resolved if 'value' in resolved else resolved.get('claim')
|
||||
if not resolved:
|
||||
raise ResolveError(f"Failed to resolve stream at '{uri}'")
|
||||
if 'error' in resolved:
|
||||
raise ResolveError(f"error resolving stream: {resolved['error']}")
|
||||
txo = response[uri]
|
||||
|
||||
claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf']))
|
||||
outpoint = f"{resolved['txid']}:{resolved['nout']}"
|
||||
resolved_time = self.loop.time() - start_time
|
||||
|
||||
# resume or update an existing stream, if the stream changed: download it and delete the old one after
|
||||
updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim)
|
||||
if updated_stream:
|
||||
log.info("already have stream for %s", uri)
|
||||
if save_file and updated_stream.output_file_exists:
|
||||
save_file = False
|
||||
await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file)
|
||||
if not updated_stream.output_file_exists and (save_file or file_name or download_directory):
|
||||
await updated_stream.save_file(
|
||||
file_name=file_name, download_directory=download_directory, node=self.node
|
||||
)
|
||||
return updated_stream
|
||||
|
||||
if not to_replace and txo.has_price and not txo.purchase_receipt:
|
||||
payment = await manager.create_purchase_transaction(
|
||||
wallet.accounts, txo, exchange_rate_manager
|
||||
)
|
||||
|
||||
stream = ManagedStream(
|
||||
self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory,
|
||||
file_name, ManagedStream.STATUS_RUNNING, content_fee=payment,
|
||||
analytics_manager=self.analytics_manager
|
||||
)
|
||||
log.info("starting download for %s", uri)
|
||||
|
||||
before_download = self.loop.time()
|
||||
await stream.start(self.node, timeout)
|
||||
stream.set_claim(resolved, claim)
|
||||
if to_replace: # delete old stream now that the replacement has started downloading
|
||||
await self.delete_stream(to_replace)
|
||||
|
||||
if payment is not None:
|
||||
await manager.broadcast_or_release(payment)
|
||||
payment = None # to avoid releasing in `finally` later
|
||||
log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri)
|
||||
await self.storage.save_content_fee(stream.stream_hash, stream.content_fee)
|
||||
|
||||
self.streams[stream.sd_hash] = stream
|
||||
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
|
||||
await self.storage.save_content_claim(stream.stream_hash, outpoint)
|
||||
if save_file:
|
||||
await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download),
|
||||
loop=self.loop)
|
||||
return stream
|
||||
except asyncio.TimeoutError:
|
||||
error = DownloadDataTimeoutError(stream.sd_hash)
|
||||
raise error
|
||||
except Exception as err: # forgive data timeout, don't delete stream
|
||||
expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError,
|
||||
KeyFeeAboveMaxAllowedError)
|
||||
if isinstance(err, expected):
|
||||
log.warning("Failed to download %s: %s", uri, str(err))
|
||||
elif isinstance(err, asyncio.CancelledError):
|
||||
pass
|
||||
else:
|
||||
log.exception("Unexpected error downloading stream:")
|
||||
error = err
|
||||
raise
|
||||
finally:
|
||||
if payment is not None:
|
||||
# payment is set to None after broadcasting, if we're here an exception probably happened
|
||||
await manager.ledger.release_tx(payment)
|
||||
if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or
|
||||
stream.downloader.time_to_first_bytes))):
|
||||
server = self.wallet_manager.ledger.network.client.server
|
||||
self.loop.create_task(
|
||||
self.analytics_manager.send_time_to_first_bytes(
|
||||
resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id,
|
||||
uri, outpoint,
|
||||
None if not stream else len(stream.downloader.blob_downloader.active_connections),
|
||||
None if not stream else len(stream.downloader.blob_downloader.scores),
|
||||
None if not stream else len(stream.downloader.blob_downloader.connection_failures),
|
||||
False if not stream else stream.downloader.added_fixed_peers,
|
||||
self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay,
|
||||
None if not stream else stream.sd_hash,
|
||||
None if not stream else stream.downloader.time_to_descriptor,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash,
|
||||
None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length,
|
||||
None if not stream else stream.downloader.time_to_first_bytes,
|
||||
None if not error else error.__class__.__name__,
|
||||
None if not error else str(error),
|
||||
None if not server else f"{server[0]}:{server[1]}"
|
||||
)
|
||||
)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
stream = self._sources[sd_hash]
|
||||
if not stream.downloader.node:
|
||||
stream.downloader.node = self.node
|
||||
return await stream.stream_file(request)
|
||||
return await self.streams[sd_hash].stream_file(request, self.node)
|
||||
+13
-83
@@ -23,13 +23,12 @@ from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode
|
||||
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbry.extras.daemon.components import Component, WalletComponent
|
||||
from lbry.extras.daemon.components import (
|
||||
DHT_COMPONENT,
|
||||
HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
)
|
||||
from lbry.extras.daemon.componentmanager import ComponentManager
|
||||
from lbry.extras.daemon.exchange_rate_manager import (
|
||||
ExchangeRateManager, ExchangeRate, BittrexBTCFeed, BittrexUSDFeed
|
||||
ExchangeRateManager, ExchangeRate, LBRYFeed, LBRYBTCFeed
|
||||
)
|
||||
from lbry.extras.daemon.storage import SQLiteStorage
|
||||
from lbry.blob.blob_manager import BlobManager
|
||||
@@ -84,7 +83,6 @@ class AsyncioTestCase(unittest.TestCase):
|
||||
# https://bugs.python.org/issue32972
|
||||
|
||||
LOOP_SLOW_CALLBACK_DURATION = 0.2
|
||||
TIMEOUT = 120.0
|
||||
|
||||
maxDiff = None
|
||||
|
||||
@@ -138,8 +136,6 @@ class AsyncioTestCase(unittest.TestCase):
|
||||
with outcome.testPartExecutor(self, isTest=True):
|
||||
maybe_coroutine = testMethod()
|
||||
if asyncio.iscoroutine(maybe_coroutine):
|
||||
if self.TIMEOUT:
|
||||
self.loop.call_later(self.TIMEOUT, self.cancel)
|
||||
self.loop.run_until_complete(maybe_coroutine)
|
||||
outcome.expecting_failure = False
|
||||
with outcome.testPartExecutor(self):
|
||||
@@ -192,12 +188,6 @@ class AsyncioTestCase(unittest.TestCase):
|
||||
if asyncio.iscoroutine(maybe_coroutine):
|
||||
self.loop.run_until_complete(maybe_coroutine)
|
||||
|
||||
def cancel(self):
|
||||
for task in asyncio.all_tasks(self.loop):
|
||||
if not task.done():
|
||||
task.print_stack()
|
||||
task.cancel()
|
||||
|
||||
|
||||
class AdvanceTimeTestCase(AsyncioTestCase):
|
||||
|
||||
@@ -263,11 +253,6 @@ class IntegrationTestCase(AsyncioTestCase):
|
||||
lambda e: e.tx.id == txid
|
||||
)
|
||||
|
||||
def on_address_update(self, address):
|
||||
return self.ledger.on_transaction.where(
|
||||
lambda e: e.address == address
|
||||
)
|
||||
|
||||
def on_transaction_address(self, tx, address):
|
||||
return self.ledger.on_transaction.where(
|
||||
lambda e: e.tx.id == tx.id and e.address == address
|
||||
@@ -291,8 +276,8 @@ class FakeExchangeRateManager(ExchangeRateManager):
|
||||
|
||||
def get_fake_exchange_rate_manager(rates=None):
|
||||
return FakeExchangeRateManager(
|
||||
[BittrexBTCFeed(), BittrexUSDFeed()],
|
||||
rates or {'BTCLBC': 3.0, 'USDLBC': 2.0}
|
||||
[LBRYFeed(), LBRYBTCFeed()],
|
||||
rates or {'BTCLBC': 3.0, 'USDBTC': 2.0}
|
||||
)
|
||||
|
||||
|
||||
@@ -330,7 +315,6 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.server_blob_manager = None
|
||||
self.server = None
|
||||
self.reflector = None
|
||||
self.skip_libtorrent = True
|
||||
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
@@ -351,7 +335,6 @@ class CommandTestCase(IntegrationTestCase):
|
||||
server_tmp_dir = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, server_tmp_dir)
|
||||
self.server_config = Config()
|
||||
self.server_config.transaction_cache_size = 10000
|
||||
self.server_storage = SQLiteStorage(self.server_config, ':memory:')
|
||||
await self.server_storage.open()
|
||||
|
||||
@@ -398,16 +381,12 @@ class CommandTestCase(IntegrationTestCase):
|
||||
conf.blockchain_name = 'lbrycrd_regtest'
|
||||
conf.lbryum_servers = [('127.0.0.1', 50001)]
|
||||
conf.reflector_servers = [('127.0.0.1', 5566)]
|
||||
conf.fixed_peers = [('127.0.0.1', 5567)]
|
||||
conf.known_dht_nodes = []
|
||||
conf.blob_lru_cache_size = self.blob_lru_cache_size
|
||||
conf.transaction_cache_size = 10000
|
||||
conf.components_to_skip = [
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT
|
||||
]
|
||||
if self.skip_libtorrent:
|
||||
conf.components_to_skip.append(LIBTORRENT_COMPONENT)
|
||||
wallet_node.manager.config = conf
|
||||
|
||||
def wallet_maker(component_manager):
|
||||
@@ -419,7 +398,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
daemon = Daemon(conf, ComponentManager(
|
||||
conf, skip_components=conf.components_to_skip, wallet=wallet_maker,
|
||||
exchange_rate_manager=partial(ExchangeRateManagerComponent, rates={
|
||||
'BTCLBC': 1.0, 'USDLBC': 2.0
|
||||
'BTCLBC': 1.0, 'USDBTC': 2.0
|
||||
})
|
||||
))
|
||||
await daemon.initialize()
|
||||
@@ -448,9 +427,8 @@ class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
async def generate(self, blocks):
|
||||
""" Ask lbrycrd to generate some blocks and wait until ledger has them. """
|
||||
prepare = self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||
await self.blockchain.generate(blocks)
|
||||
await prepare # no guarantee that it didn't happen already, so start waiting from before calling generate
|
||||
await self.ledger.on_header.where(self.blockchain.is_expected_block)
|
||||
|
||||
async def blockchain_claim_name(self, name: str, value: str, amount: str, confirm=True):
|
||||
txid = await self.blockchain._cli_cmnd('claimname', name, value, amount)
|
||||
@@ -490,7 +468,7 @@ class CommandTestCase(IntegrationTestCase):
|
||||
async def stream_create(
|
||||
self, name='hovercraft', bid='1.0', file_path=None,
|
||||
data=b'hi!', confirm=True, prefix=None, suffix=None, **kwargs):
|
||||
if file_path is None and data is not None:
|
||||
if file_path is None:
|
||||
file_path = self.create_upload_file(data=data, prefix=prefix, suffix=suffix)
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs), confirm
|
||||
@@ -507,8 +485,8 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.daemon.jsonrpc_stream_update(claim_id, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
def stream_repost(self, claim_id, name='repost', bid='1.0', confirm=True, **kwargs):
|
||||
return self.confirm_and_render(
|
||||
self.daemon.jsonrpc_stream_repost(claim_id=claim_id, name=name, bid=bid, **kwargs), confirm
|
||||
)
|
||||
|
||||
@@ -519,11 +497,6 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.daemon.jsonrpc_stream_abandon(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def purchase_create(self, *args, confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_purchase_create(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def publish(self, name, *args, confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_publish(name, *args, **kwargs), confirm
|
||||
@@ -570,33 +543,8 @@ class CommandTestCase(IntegrationTestCase):
|
||||
self.daemon.jsonrpc_support_create(claim_id, bid, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def support_abandon(self, *args, confirm=True, **kwargs):
|
||||
if 'blocking' not in kwargs:
|
||||
kwargs['blocking'] = False
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_support_abandon(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def account_send(self, *args, confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_account_send(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def wallet_send(self, *args, confirm=True, **kwargs):
|
||||
return await self.confirm_and_render(
|
||||
self.daemon.jsonrpc_wallet_send(*args, **kwargs), confirm
|
||||
)
|
||||
|
||||
async def txo_spend(self, *args, confirm=True, **kwargs):
|
||||
txs = await self.daemon.jsonrpc_txo_spend(*args, **kwargs)
|
||||
if confirm:
|
||||
await asyncio.wait([self.ledger.wait(tx) for tx in txs])
|
||||
await self.generate(1)
|
||||
await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs])
|
||||
return self.sout(txs)
|
||||
|
||||
async def resolve(self, uri, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri]
|
||||
async def resolve(self, uri):
|
||||
return await self.out(self.daemon.jsonrpc_resolve(uri))
|
||||
|
||||
async def claim_search(self, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_claim_search(**kwargs)))['items']
|
||||
@@ -604,30 +552,12 @@ class CommandTestCase(IntegrationTestCase):
|
||||
async def file_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_file_list(*args, **kwargs)))['items']
|
||||
|
||||
async def txo_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_txo_list(*args, **kwargs)))['items']
|
||||
|
||||
async def txo_sum(self, *args, **kwargs):
|
||||
return await self.out(self.daemon.jsonrpc_txo_sum(*args, **kwargs))
|
||||
|
||||
async def txo_plot(self, *args, **kwargs):
|
||||
return await self.out(self.daemon.jsonrpc_txo_plot(*args, **kwargs))
|
||||
|
||||
async def claim_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_claim_list(*args, **kwargs)))['items']
|
||||
|
||||
async def stream_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_stream_list(*args, **kwargs)))['items']
|
||||
|
||||
async def channel_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_channel_list(*args, **kwargs)))['items']
|
||||
|
||||
async def transaction_list(self, *args, **kwargs):
|
||||
return (await self.out(self.daemon.jsonrpc_transaction_list(*args, **kwargs)))['items']
|
||||
|
||||
@staticmethod
|
||||
def get_claim_id(tx):
|
||||
return tx['outputs'][0]['claim_id']
|
||||
|
||||
def assertItemCount(self, result, count): # pylint: disable=invalid-name
|
||||
self.assertEqual(count, result['total_items'])
|
||||
self.assertEqual(result['total_items'], count)
|
||||
Whitespace-only changes.
@@ -1,290 +0,0 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import os
|
||||
import logging
|
||||
import random
|
||||
from hashlib import sha1
|
||||
from tempfile import mkdtemp
|
||||
from typing import Optional
|
||||
|
||||
import libtorrent
|
||||
|
||||
|
||||
NOTIFICATION_MASKS = [
|
||||
"error",
|
||||
"peer",
|
||||
"port_mapping",
|
||||
"storage",
|
||||
"tracker",
|
||||
"debug",
|
||||
"status",
|
||||
"progress",
|
||||
"ip_block",
|
||||
"dht",
|
||||
"stats",
|
||||
"session_log",
|
||||
"torrent_log",
|
||||
"peer_log",
|
||||
"incoming_request",
|
||||
"dht_log",
|
||||
"dht_operation",
|
||||
"port_mapping_log",
|
||||
"picker_log",
|
||||
"file_progress",
|
||||
"piece_progress",
|
||||
"upload",
|
||||
"block_progress"
|
||||
]
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted?
|
||||
libtorrent.add_torrent_params_flags_t.flag_auto_managed
|
||||
| libtorrent.add_torrent_params_flags_t.flag_update_subscribe
|
||||
)
|
||||
|
||||
|
||||
def get_notification_type(notification) -> str:
|
||||
for i, notification_type in enumerate(NOTIFICATION_MASKS):
|
||||
if (1 << i) & notification:
|
||||
return notification_type
|
||||
raise ValueError("unrecognized notification type")
|
||||
|
||||
|
||||
class TorrentHandle:
|
||||
def __init__(self, loop, executor, handle):
|
||||
self._loop = loop
|
||||
self._executor = executor
|
||||
self._handle: libtorrent.torrent_handle = handle
|
||||
self.started = asyncio.Event(loop=loop)
|
||||
self.finished = asyncio.Event(loop=loop)
|
||||
self.metadata_completed = asyncio.Event(loop=loop)
|
||||
self.size = 0
|
||||
self.total_wanted_done = 0
|
||||
self.name = ''
|
||||
self.tasks = []
|
||||
self.torrent_file: Optional[libtorrent.file_storage] = None
|
||||
self._base_path = None
|
||||
self._handle.set_sequential_download(1)
|
||||
|
||||
@property
|
||||
def largest_file(self) -> Optional[str]:
|
||||
if not self.torrent_file:
|
||||
return None
|
||||
index = self.largest_file_index
|
||||
return os.path.join(self._base_path, self.torrent_file.at(index).path)
|
||||
|
||||
@property
|
||||
def largest_file_index(self):
|
||||
largest_size, index = 0, 0
|
||||
for file_num in range(self.torrent_file.num_files()):
|
||||
if self.torrent_file.file_size(file_num) > largest_size:
|
||||
largest_size = self.torrent_file.file_size(file_num)
|
||||
index = file_num
|
||||
return index
|
||||
|
||||
def stop_tasks(self):
|
||||
while self.tasks:
|
||||
self.tasks.pop().cancel()
|
||||
|
||||
def _show_status(self):
|
||||
# fixme: cleanup
|
||||
if not self._handle.is_valid():
|
||||
return
|
||||
status = self._handle.status()
|
||||
if status.has_metadata:
|
||||
self.size = status.total_wanted
|
||||
self.total_wanted_done = status.total_wanted_done
|
||||
self.name = status.name
|
||||
if not self.metadata_completed.is_set():
|
||||
self.metadata_completed.set()
|
||||
log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name)
|
||||
self.torrent_file = self._handle.get_torrent_info().files()
|
||||
self._base_path = status.save_path
|
||||
first_piece = self.torrent_file.at(self.largest_file_index).offset
|
||||
if not self.started.is_set():
|
||||
if self._handle.have_piece(first_piece):
|
||||
self.started.set()
|
||||
else:
|
||||
# prioritize it
|
||||
self._handle.set_piece_deadline(first_piece, 100)
|
||||
if not status.is_seeding:
|
||||
log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s',
|
||||
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
|
||||
status.num_peers, status.num_seeds, status.state, status.save_path)
|
||||
elif not self.finished.is_set():
|
||||
self.finished.set()
|
||||
log.info("Torrent finished: %s", self.name)
|
||||
|
||||
async def status_loop(self):
|
||||
while True:
|
||||
self._show_status()
|
||||
if self.finished.is_set():
|
||||
break
|
||||
await asyncio.sleep(0.1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._handle.pause
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._handle.resume() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
|
||||
|
||||
class TorrentSession:
|
||||
def __init__(self, loop, executor):
|
||||
self._loop = loop
|
||||
self._executor = executor
|
||||
self._session: Optional[libtorrent.session] = None
|
||||
self._handles = {}
|
||||
self.tasks = []
|
||||
self.wait_start = True
|
||||
|
||||
async def add_fake_torrent(self):
|
||||
tmpdir = mkdtemp()
|
||||
info, btih = _create_fake_torrent(tmpdir)
|
||||
flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode
|
||||
handle = self._session.add_torrent({
|
||||
'ti': info, 'save_path': tmpdir, 'flags': flags
|
||||
})
|
||||
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
|
||||
return btih
|
||||
|
||||
async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
|
||||
settings = {
|
||||
'listen_interfaces': f"{interface}:{port}",
|
||||
'enable_outgoing_utp': True,
|
||||
'enable_incoming_utp': True,
|
||||
'enable_outgoing_tcp': False,
|
||||
'enable_incoming_tcp': False
|
||||
}
|
||||
self._session = await self._loop.run_in_executor(
|
||||
self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member
|
||||
)
|
||||
self.tasks.append(self._loop.create_task(self.process_alerts()))
|
||||
|
||||
def stop(self):
|
||||
while self.tasks:
|
||||
self.tasks.pop().cancel()
|
||||
self._session.save_state()
|
||||
self._session.pause()
|
||||
self._session.stop_dht()
|
||||
self._session.stop_lsd()
|
||||
self._session.stop_natpmp()
|
||||
self._session.stop_upnp()
|
||||
self._session = None
|
||||
|
||||
def _pop_alerts(self):
|
||||
for alert in self._session.pop_alerts():
|
||||
log.info("torrent alert: %s", alert)
|
||||
|
||||
async def process_alerts(self):
|
||||
while True:
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._pop_alerts
|
||||
)
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._session.save_state() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, lambda: self._session.pause() # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._session.resume
|
||||
)
|
||||
|
||||
def _add_torrent(self, btih: str, download_directory: Optional[str]):
|
||||
params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS}
|
||||
if download_directory:
|
||||
params['save_path'] = download_directory
|
||||
handle = self._session.add_torrent(params)
|
||||
handle.force_dht_announce()
|
||||
self._handles[btih] = TorrentHandle(self._loop, self._executor, handle)
|
||||
|
||||
def full_path(self, btih):
|
||||
return self._handles[btih].largest_file
|
||||
|
||||
async def add_torrent(self, btih, download_path):
|
||||
await self._loop.run_in_executor(
|
||||
self._executor, self._add_torrent, btih, download_path
|
||||
)
|
||||
self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop()))
|
||||
await self._handles[btih].metadata_completed.wait()
|
||||
if self.wait_start:
|
||||
# fixme: temporary until we add streaming support, otherwise playback fails!
|
||||
await self._handles[btih].started.wait()
|
||||
|
||||
def remove_torrent(self, btih, remove_files=False):
|
||||
if btih in self._handles:
|
||||
handle = self._handles[btih]
|
||||
handle.stop_tasks()
|
||||
self._session.remove_torrent(handle._handle, 1 if remove_files else 0)
|
||||
self._handles.pop(btih)
|
||||
|
||||
async def save_file(self, btih, download_directory):
|
||||
handle = self._handles[btih]
|
||||
await handle.resume()
|
||||
|
||||
def get_size(self, btih):
|
||||
return self._handles[btih].size
|
||||
|
||||
def get_name(self, btih):
|
||||
return self._handles[btih].name
|
||||
|
||||
def get_downloaded(self, btih):
|
||||
return self._handles[btih].total_wanted_done
|
||||
|
||||
def is_completed(self, btih):
|
||||
return self._handles[btih].finished.is_set()
|
||||
|
||||
|
||||
def get_magnet_uri(btih):
|
||||
return f"magnet:?xt=urn:btih:{btih}"
|
||||
|
||||
|
||||
def _create_fake_torrent(tmpdir):
|
||||
# beware, that's just for testing
|
||||
path = os.path.join(tmpdir, 'tmp')
|
||||
with open(path, 'wb') as myfile:
|
||||
size = myfile.write(bytes([random.randint(0, 255) for _ in range(40)]) * 1024)
|
||||
file_storage = libtorrent.file_storage()
|
||||
file_storage.add_file('tmp', size)
|
||||
t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024)
|
||||
libtorrent.set_piece_hashes(t, tmpdir)
|
||||
info = libtorrent.torrent_info(t.generate())
|
||||
btih = sha1(info.metadata()).hexdigest()
|
||||
return info, btih
|
||||
|
||||
|
||||
async def main():
|
||||
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent"):
|
||||
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent")
|
||||
if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso"):
|
||||
os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso")
|
||||
|
||||
btih = "dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c"
|
||||
|
||||
executor = None
|
||||
session = TorrentSession(asyncio.get_event_loop(), executor)
|
||||
session2 = TorrentSession(asyncio.get_event_loop(), executor)
|
||||
await session.bind('localhost', port=4040)
|
||||
await session2.bind('localhost', port=4041)
|
||||
btih = await session.add_fake_torrent()
|
||||
session2._session.add_dht_node(('localhost', 4040))
|
||||
await session2.add_torrent(btih, "/tmp/down")
|
||||
while True:
|
||||
await asyncio.sleep(100)
|
||||
await session.pause()
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,72 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import typing
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TorrentInfo:
|
||||
__slots__ = ('dht_seeds', 'http_seeds', 'trackers', 'total_size')
|
||||
|
||||
def __init__(self, dht_seeds: typing.Tuple[typing.Tuple[str, int]],
|
||||
http_seeds: typing.Tuple[typing.Dict[str, typing.Any]],
|
||||
trackers: typing.Tuple[typing.Tuple[str, int]], total_size: int):
|
||||
self.dht_seeds = dht_seeds
|
||||
self.http_seeds = http_seeds
|
||||
self.trackers = trackers
|
||||
self.total_size = total_size
|
||||
|
||||
@classmethod
|
||||
def from_libtorrent_info(cls, torrent_info):
|
||||
return cls(
|
||||
torrent_info.nodes(), tuple(
|
||||
{
|
||||
'url': web_seed['url'],
|
||||
'type': web_seed['type'],
|
||||
'auth': web_seed['auth']
|
||||
} for web_seed in torrent_info.web_seeds()
|
||||
), tuple(
|
||||
(tracker.url, tracker.tier) for tracker in torrent_info.trackers()
|
||||
), torrent_info.total_size()
|
||||
)
|
||||
|
||||
|
||||
class Torrent:
|
||||
def __init__(self, loop, handle):
|
||||
self._loop = loop
|
||||
self._handle = handle
|
||||
self.finished = asyncio.Event(loop=loop)
|
||||
|
||||
def _threaded_update_status(self):
|
||||
status = self._handle.status()
|
||||
if not status.is_seeding:
|
||||
log.info(
|
||||
'%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s',
|
||||
status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000,
|
||||
status.num_peers, status.state
|
||||
)
|
||||
elif not self.finished.is_set():
|
||||
self.finished.set()
|
||||
|
||||
async def wait_for_finished(self):
|
||||
while True:
|
||||
await self._loop.run_in_executor(
|
||||
None, self._threaded_update_status
|
||||
)
|
||||
if self.finished.is_set():
|
||||
log.info("finished downloading torrent!")
|
||||
await self.pause()
|
||||
break
|
||||
await asyncio.sleep(1, loop=self._loop)
|
||||
|
||||
async def pause(self):
|
||||
log.info("pause torrent")
|
||||
await self._loop.run_in_executor(
|
||||
None, self._handle.pause
|
||||
)
|
||||
|
||||
async def resume(self):
|
||||
await self._loop.run_in_executor(
|
||||
None, self._handle.resume
|
||||
)
|
||||
@@ -1,140 +0,0 @@
|
||||
import asyncio
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import typing
|
||||
from typing import Optional
|
||||
from aiohttp.web import Request
|
||||
from lbry.file.source_manager import SourceManager
|
||||
from lbry.file.source import ManagedDownloadSource
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from lbry.torrent.session import TorrentSession
|
||||
from lbry.conf import Config
|
||||
from lbry.wallet.transaction import Transaction
|
||||
from lbry.extras.daemon.analytics import AnalyticsManager
|
||||
from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim
|
||||
from lbry.extras.daemon.storage import StoredContentClaim
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def path_or_none(encoded_path) -> Optional[str]:
|
||||
if not encoded_path:
|
||||
return
|
||||
return binascii.unhexlify(encoded_path).decode()
|
||||
|
||||
|
||||
class TorrentSource(ManagedDownloadSource):
|
||||
STATUS_STOPPED = "stopped"
|
||||
filter_fields = SourceManager.filter_fields
|
||||
filter_fields.update({
|
||||
'bt_infohash'
|
||||
})
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str,
|
||||
file_name: Optional[str] = None, download_directory: Optional[str] = None,
|
||||
status: Optional[str] = STATUS_STOPPED, claim: Optional['StoredContentClaim'] = None,
|
||||
download_id: Optional[str] = None, rowid: Optional[int] = None,
|
||||
content_fee: Optional['Transaction'] = None,
|
||||
analytics_manager: Optional['AnalyticsManager'] = None,
|
||||
added_on: Optional[int] = None, torrent_session: Optional['TorrentSession'] = None):
|
||||
super().__init__(loop, config, storage, identifier, file_name, download_directory, status, claim, download_id,
|
||||
rowid, content_fee, analytics_manager, added_on)
|
||||
self.torrent_session = torrent_session
|
||||
|
||||
@property
|
||||
def full_path(self) -> Optional[str]:
|
||||
full_path = self.torrent_session.full_path(self.identifier)
|
||||
self.download_directory = os.path.dirname(full_path)
|
||||
return full_path
|
||||
|
||||
async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False):
|
||||
await self.torrent_session.add_torrent(self.identifier, self.download_directory)
|
||||
|
||||
async def stop(self, finished: bool = False):
|
||||
await self.torrent_session.remove_torrent(self.identifier)
|
||||
|
||||
async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None):
|
||||
await self.torrent_session.save_file(self.identifier, download_directory)
|
||||
|
||||
@property
|
||||
def torrent_length(self):
|
||||
return self.torrent_session.get_size(self.identifier)
|
||||
|
||||
@property
|
||||
def written_bytes(self):
|
||||
return self.torrent_session.get_downloaded(self.identifier)
|
||||
|
||||
@property
|
||||
def torrent_name(self):
|
||||
return self.torrent_session.get_name(self.identifier)
|
||||
|
||||
@property
|
||||
def bt_infohash(self):
|
||||
return self.identifier
|
||||
|
||||
def stop_tasks(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def completed(self):
|
||||
return self.torrent_session.is_completed(self.identifier)
|
||||
|
||||
|
||||
class TorrentManager(SourceManager):
|
||||
_sources: typing.Dict[str, ManagedDownloadSource]
|
||||
|
||||
filter_fields = set(SourceManager.filter_fields)
|
||||
filter_fields.update({
|
||||
'bt_infohash',
|
||||
'blobs_remaining', # TODO: here they call them "parts", but its pretty much the same concept
|
||||
'blobs_in_stream'
|
||||
})
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', torrent_session: 'TorrentSession',
|
||||
storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None):
|
||||
super().__init__(loop, config, storage, analytics_manager)
|
||||
self.torrent_session: 'TorrentSession' = torrent_session
|
||||
|
||||
async def recover_streams(self, file_infos: typing.List[typing.Dict]):
|
||||
raise NotImplementedError
|
||||
|
||||
async def _load_stream(self, rowid: int, bt_infohash: str, file_name: Optional[str],
|
||||
download_directory: Optional[str], status: str,
|
||||
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
|
||||
added_on: Optional[int]):
|
||||
stream = TorrentSource(
|
||||
self.loop, self.config, self.storage, identifier=bt_infohash, file_name=file_name,
|
||||
download_directory=download_directory, status=status, claim=claim, rowid=rowid,
|
||||
content_fee=content_fee, analytics_manager=self.analytics_manager, added_on=added_on,
|
||||
torrent_session=self.torrent_session
|
||||
)
|
||||
self.add(stream)
|
||||
|
||||
async def initialize_from_database(self):
|
||||
pass
|
||||
|
||||
async def start(self):
|
||||
await super().start()
|
||||
|
||||
def stop(self):
|
||||
super().stop()
|
||||
log.info("finished stopping the torrent manager")
|
||||
|
||||
async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
await super().delete(source, delete_file)
|
||||
self.torrent_session.remove_torrent(source.identifier, delete_file)
|
||||
|
||||
async def create(self, file_path: str, key: Optional[bytes] = None,
|
||||
iv_generator: Optional[typing.Generator[bytes, None, None]] = None):
|
||||
raise NotImplementedError
|
||||
|
||||
async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False):
|
||||
raise NotImplementedError
|
||||
# blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]]
|
||||
# await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False)
|
||||
# await self.storage.delete_stream(source.descriptor)
|
||||
|
||||
async def stream_partial_content(self, request: Request, sd_hash: str):
|
||||
raise NotImplementedError
|
||||
+13
-215
@@ -3,9 +3,7 @@ import codecs
|
||||
import datetime
|
||||
import random
|
||||
import socket
|
||||
import time
|
||||
import string
|
||||
import sys
|
||||
import json
|
||||
import typing
|
||||
import asyncio
|
||||
@@ -20,11 +18,8 @@ import pkg_resources
|
||||
|
||||
import certifi
|
||||
import aiohttp
|
||||
from prometheus_client import Counter
|
||||
from prometheus_client.registry import REGISTRY
|
||||
from lbry.schema.claim import Claim
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -105,6 +100,10 @@ def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def async_check_connection(server="lbry.com", port=80, timeout=1) -> bool:
|
||||
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
|
||||
|
||||
|
||||
def random_string(length=10, chars=string.ascii_lowercase):
|
||||
return ''.join([random.choice(chars) for _ in range(length)])
|
||||
|
||||
@@ -188,8 +187,6 @@ def cache_concurrent(async_fn):
|
||||
async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
if proto not in ['udp', 'tcp']:
|
||||
raise Exception("invalid protocol")
|
||||
if url.lower() == 'localhost':
|
||||
return '127.0.0.1'
|
||||
try:
|
||||
if ipaddress.ip_address(url):
|
||||
return url
|
||||
@@ -204,101 +201,18 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
|
||||
))[0][4][0]
|
||||
|
||||
|
||||
class LRUCacheWithMetrics:
|
||||
__slots__ = [
|
||||
'capacity',
|
||||
'cache',
|
||||
'_track_metrics',
|
||||
'hits',
|
||||
'misses'
|
||||
]
|
||||
|
||||
def __init__(self, capacity: int, metric_name: typing.Optional[str] = None, namespace: str = "daemon_cache"):
|
||||
self.capacity = capacity
|
||||
self.cache = collections.OrderedDict()
|
||||
if metric_name is None:
|
||||
self._track_metrics = False
|
||||
self.hits = self.misses = None
|
||||
else:
|
||||
self._track_metrics = True
|
||||
try:
|
||||
self.hits = Counter(
|
||||
f"{metric_name}_cache_hit_count", "Number of cache hits", namespace=namespace
|
||||
)
|
||||
self.misses = Counter(
|
||||
f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
|
||||
)
|
||||
except ValueError as err:
|
||||
log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
|
||||
self._track_metrics = False
|
||||
self.hits = self.misses = None
|
||||
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
value = self.cache.pop(key)
|
||||
if self._track_metrics:
|
||||
self.hits.inc()
|
||||
except KeyError:
|
||||
if self._track_metrics:
|
||||
self.misses.inc()
|
||||
return default
|
||||
self.cache[key] = value
|
||||
return value
|
||||
|
||||
def set(self, key, value):
|
||||
try:
|
||||
self.cache.pop(key)
|
||||
except KeyError:
|
||||
if len(self.cache) >= self.capacity:
|
||||
self.cache.popitem(last=False)
|
||||
self.cache[key] = value
|
||||
|
||||
def clear(self):
|
||||
self.cache.clear()
|
||||
|
||||
def pop(self, key):
|
||||
return self.cache.pop(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.set(key, value)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.get(item)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.cache
|
||||
|
||||
def __len__(self):
|
||||
return len(self.cache)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.cache.pop(key)
|
||||
|
||||
def __del__(self):
|
||||
self.clear()
|
||||
if self._track_metrics: # needed for tests
|
||||
try:
|
||||
REGISTRY.unregister(self.hits)
|
||||
REGISTRY.unregister(self.misses)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
class LRUCache:
|
||||
__slots__ = [
|
||||
'capacity',
|
||||
'cache'
|
||||
]
|
||||
|
||||
def __init__(self, capacity: int):
|
||||
def __init__(self, capacity):
|
||||
self.capacity = capacity
|
||||
self.cache = collections.OrderedDict()
|
||||
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
value = self.cache.pop(key)
|
||||
except KeyError:
|
||||
return default
|
||||
def get(self, key):
|
||||
value = self.cache.pop(key)
|
||||
self.cache[key] = value
|
||||
return value
|
||||
|
||||
@@ -310,37 +224,16 @@ class LRUCache:
|
||||
self.cache.popitem(last=False)
|
||||
self.cache[key] = value
|
||||
|
||||
def clear(self):
|
||||
self.cache.clear()
|
||||
|
||||
def pop(self, key):
|
||||
return self.cache.pop(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
return self.set(key, value)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.get(item)
|
||||
|
||||
def __contains__(self, item) -> bool:
|
||||
return item in self.cache
|
||||
|
||||
def __len__(self):
|
||||
return len(self.cache)
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.cache.pop(key)
|
||||
|
||||
def __del__(self):
|
||||
self.clear()
|
||||
|
||||
|
||||
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
|
||||
override_lru_cache: typing.Optional[LRUCacheWithMetrics] = None):
|
||||
override_lru_cache: typing.Optional[LRUCache] = None):
|
||||
if not cache_size and override_lru_cache is None:
|
||||
raise ValueError("invalid cache size")
|
||||
concurrent_cache = {}
|
||||
lru_cache = override_lru_cache if override_lru_cache is not None else LRUCacheWithMetrics(cache_size)
|
||||
lru_cache = override_lru_cache or LRUCache(cache_size)
|
||||
|
||||
def wrapper(async_fn):
|
||||
|
||||
@@ -371,110 +264,15 @@ def get_ssl_context() -> ssl.SSLContext:
|
||||
@contextlib.asynccontextmanager
|
||||
async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[aiohttp.ClientResponse]:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.request(method, url, **kwargs) as response:
|
||||
async with session.request(method, url, ssl=get_ssl_context(), **kwargs) as response:
|
||||
yield response
|
||||
|
||||
|
||||
# the ipaddress module does not show these subnets as reserved
|
||||
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
|
||||
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
|
||||
|
||||
|
||||
def is_valid_public_ipv4(address, allow_localhost: bool = False, allow_lan: bool = False):
|
||||
try:
|
||||
parsed_ip = ipaddress.ip_address(address)
|
||||
if parsed_ip.is_loopback and allow_localhost:
|
||||
return True
|
||||
if allow_lan and parsed_ip.is_private:
|
||||
return True
|
||||
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
|
||||
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private)):
|
||||
return False
|
||||
else:
|
||||
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
|
||||
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
|
||||
except (ipaddress.AddressValueError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
async def fallback_get_external_ip(): # used if spv servers can't be used for ip detection
|
||||
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip'], None
|
||||
return response['data']['ip']
|
||||
except Exception:
|
||||
return None, None
|
||||
|
||||
|
||||
async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||
# used if upnp is disabled or non-functioning
|
||||
from lbry.wallet.server.udp import SPVStatusClientProtocol # pylint: disable=C0415
|
||||
|
||||
hostname_to_ip = {}
|
||||
ip_to_hostnames = collections.defaultdict(list)
|
||||
|
||||
async def resolve_spv(server, port):
|
||||
try:
|
||||
server_addr = await resolve_host(server, port, 'udp')
|
||||
hostname_to_ip[server] = (server_addr, port)
|
||||
ip_to_hostnames[(server_addr, port)].append(server)
|
||||
except Exception:
|
||||
log.exception("error looking up dns for spv servers")
|
||||
|
||||
# accumulate the dns results
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers))
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
pong_responses = asyncio.Queue()
|
||||
connection = SPVStatusClientProtocol(pong_responses)
|
||||
try:
|
||||
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||
# could raise OSError if it cant bind
|
||||
randomized_servers = list(ip_to_hostnames.keys())
|
||||
random.shuffle(randomized_servers)
|
||||
for server in randomized_servers:
|
||||
connection.ping(server)
|
||||
try:
|
||||
_, pong = await asyncio.wait_for(pong_responses.get(), 1)
|
||||
if is_valid_public_ipv4(pong.ip_address):
|
||||
return pong.ip_address, ip_to_hostnames[server][0]
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
return None, None
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
|
||||
ip_from_spv_servers = await _get_external_ip(default_servers)
|
||||
if not ip_from_spv_servers[1]:
|
||||
return await fallback_get_external_ip()
|
||||
return ip_from_spv_servers
|
||||
|
||||
|
||||
def is_running_from_bundle():
|
||||
# see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||
return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
|
||||
|
||||
|
||||
class LockWithMetrics(asyncio.Lock):
|
||||
def __init__(self, acquire_metric, held_time_metric, loop=None):
|
||||
super().__init__(loop=loop)
|
||||
self._acquire_metric = acquire_metric
|
||||
self._lock_held_time_metric = held_time_metric
|
||||
self._lock_acquired_time = None
|
||||
|
||||
async def acquire(self):
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
return await super().acquire()
|
||||
finally:
|
||||
self._lock_acquired_time = time.perf_counter()
|
||||
self._acquire_metric.observe(self._lock_acquired_time - start)
|
||||
|
||||
def release(self):
|
||||
try:
|
||||
return super().release()
|
||||
finally:
|
||||
self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time)
|
||||
return
|
||||
@@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd'
|
||||
__node_cli__ = 'lbrycrd-cli'
|
||||
__node_bin__ = ''
|
||||
__node_url__ = (
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.6/lbrycrd-linux-1746.zip'
|
||||
'https://github.com/lbryio/lbrycrd/releases/download/v0.17.3.2/lbrycrd-linux-1732.zip'
|
||||
)
|
||||
__spvserver__ = 'lbry.wallet.server.coin.LBCRegTest'
|
||||
|
||||
|
||||
+48
-41
@@ -5,6 +5,7 @@ import logging
|
||||
import typing
|
||||
import asyncio
|
||||
import random
|
||||
from functools import partial
|
||||
from hashlib import sha256
|
||||
from string import hexdigits
|
||||
from typing import Type, Dict, Tuple, Optional, Any, List
|
||||
@@ -15,7 +16,7 @@ from lbry.crypto.crypt import aes_encrypt, aes_decrypt
|
||||
|
||||
from .bip32 import PrivateKey, PubKey, from_extended_key_string
|
||||
from .mnemonic import Mnemonic
|
||||
from .constants import COIN, TXO_TYPES
|
||||
from .constants import COIN, CLAIM_TYPES, TXO_TYPES
|
||||
from .transaction import Transaction, Input, Output
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -70,7 +71,6 @@ class AddressManager:
|
||||
|
||||
def _query_addresses(self, **constraints):
|
||||
return self.account.ledger.db.get_addresses(
|
||||
read_only=constraints.pop("read_only", False),
|
||||
accounts=[self.account],
|
||||
chain=self.chain_number,
|
||||
**constraints
|
||||
@@ -310,7 +310,7 @@ class Account:
|
||||
private_key=private_key,
|
||||
public_key=public_key,
|
||||
address_generator=d.get('address_generator', {}),
|
||||
modified_on=int(d.get('modified_on', time.time())),
|
||||
modified_on=d.get('modified_on', time.time()),
|
||||
channel_keys=d.get('certificates', {})
|
||||
)
|
||||
|
||||
@@ -342,7 +342,7 @@ class Account:
|
||||
def merge(self, d: dict):
|
||||
if d.get('modified_on', 0) > self.modified_on:
|
||||
self.name = d['name']
|
||||
self.modified_on = int(d.get('modified_on', time.time()))
|
||||
self.modified_on = d.get('modified_on', time.time())
|
||||
assert self.address_generator.name == d['address_generator']['name']
|
||||
for chain_name in ('change', 'receiving'):
|
||||
if chain_name in d['address_generator']:
|
||||
@@ -434,9 +434,9 @@ class Account:
|
||||
addresses.extend(new_addresses)
|
||||
return addresses
|
||||
|
||||
async def get_addresses(self, read_only=False, **constraints) -> List[str]:
|
||||
rows = await self.ledger.db.select_addresses('address', read_only=read_only, accounts=[self], **constraints)
|
||||
return [r['address'] for r in rows]
|
||||
async def get_addresses(self, **constraints) -> List[str]:
|
||||
rows = await self.ledger.db.select_addresses('address', accounts=[self], **constraints)
|
||||
return [r[0] for r in rows]
|
||||
|
||||
def get_address_records(self, **constraints):
|
||||
return self.ledger.db.get_addresses(accounts=[self], **constraints)
|
||||
@@ -451,13 +451,13 @@ class Account:
|
||||
def get_public_key(self, chain: int, index: int) -> PubKey:
|
||||
return self.address_managers[chain].get_public_key(index)
|
||||
|
||||
def get_balance(self, confirmations=0, include_claims=False, read_only=False, **constraints):
|
||||
def get_balance(self, confirmations: int = 0, include_claims=False, **constraints):
|
||||
if not include_claims:
|
||||
constraints.update({'txo_type__in': (TXO_TYPES['other'], TXO_TYPES['purchase'])})
|
||||
constraints.update({'txo_type__in': (0, TXO_TYPES['purchase'])})
|
||||
if confirmations > 0:
|
||||
height = self.ledger.headers.height - (confirmations-1)
|
||||
constraints.update({'height__lte': height, 'height__gt': 0})
|
||||
return self.ledger.db.get_balance(accounts=[self], read_only=read_only, **constraints)
|
||||
return self.ledger.db.get_balance(accounts=[self], **constraints)
|
||||
|
||||
async def get_max_gap(self):
|
||||
change_gap = await self.change.get_max_gap()
|
||||
@@ -467,12 +467,6 @@ class Account:
|
||||
'max_receiving_gap': receiving_gap,
|
||||
}
|
||||
|
||||
def get_txos(self, **constraints):
|
||||
return self.ledger.get_txos(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_txo_count(self, **constraints):
|
||||
return self.ledger.get_txo_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_utxos(self, **constraints):
|
||||
return self.ledger.get_utxos(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
@@ -524,18 +518,13 @@ class Account:
|
||||
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
||||
self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode()
|
||||
|
||||
async def get_channel_private_key(self, public_key_bytes):
|
||||
def get_channel_private_key(self, public_key_bytes):
|
||||
channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes)
|
||||
private_key_pem = self.channel_keys.get(channel_pubkey_hash)
|
||||
if private_key_pem:
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
None, ecdsa.SigningKey.from_pem, private_key_pem, sha256
|
||||
)
|
||||
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
|
||||
|
||||
async def maybe_migrate_certificates(self):
|
||||
def to_der(private_key_pem):
|
||||
return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256).get_verifying_key().to_der()
|
||||
|
||||
if not self.channel_keys:
|
||||
return
|
||||
channel_keys = {}
|
||||
@@ -544,7 +533,8 @@ class Account:
|
||||
continue
|
||||
if "-----BEGIN EC PRIVATE KEY-----" not in private_key_pem:
|
||||
continue
|
||||
public_key_der = await asyncio.get_event_loop().run_in_executor(None, to_der, private_key_pem)
|
||||
private_key = ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256)
|
||||
public_key_der = private_key.get_verifying_key().to_der()
|
||||
channel_keys[self.ledger.public_key_to_address(public_key_der)] = private_key_pem
|
||||
if self.channel_keys != channel_keys:
|
||||
self.channel_keys = channel_keys
|
||||
@@ -565,24 +555,41 @@ class Account:
|
||||
if gap_changed:
|
||||
self.wallet.save()
|
||||
|
||||
async def get_detailed_balance(self, confirmations=0, read_only=False):
|
||||
constraints = {}
|
||||
if confirmations > 0:
|
||||
height = self.ledger.headers.height - (confirmations-1)
|
||||
constraints.update({'height__lte': height, 'height__gt': 0})
|
||||
return await self.ledger.db.get_detailed_balance(
|
||||
accounts=[self], read_only=read_only, **constraints
|
||||
)
|
||||
async def get_detailed_balance(self, confirmations=0, reserved_subtotals=False):
|
||||
tips_balance, supports_balance, claims_balance = 0, 0, 0
|
||||
get_total_balance = partial(self.get_balance, confirmations=confirmations, include_claims=True)
|
||||
total = await get_total_balance()
|
||||
if reserved_subtotals:
|
||||
claims_balance = await get_total_balance(txo_type__in=CLAIM_TYPES)
|
||||
for amount, spent, from_me, to_me, height in await self.get_support_summary():
|
||||
if confirmations > 0 and not 0 < height <= self.ledger.headers.height - (confirmations - 1):
|
||||
continue
|
||||
if not spent and to_me:
|
||||
if from_me:
|
||||
supports_balance += amount
|
||||
else:
|
||||
tips_balance += amount
|
||||
reserved = claims_balance + supports_balance + tips_balance
|
||||
else:
|
||||
reserved = await self.get_balance(
|
||||
confirmations=confirmations, include_claims=True, txo_type__gt=0
|
||||
)
|
||||
return {
|
||||
'total': total,
|
||||
'available': total - reserved,
|
||||
'reserved': reserved,
|
||||
'reserved_subtotals': {
|
||||
'claims': claims_balance,
|
||||
'supports': supports_balance,
|
||||
'tips': tips_balance
|
||||
} if reserved_subtotals else None
|
||||
}
|
||||
|
||||
def get_transaction_history(self, read_only=False, **constraints):
|
||||
return self.ledger.get_transaction_history(
|
||||
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
|
||||
)
|
||||
def get_transaction_history(self, **constraints):
|
||||
return self.ledger.get_transaction_history(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_transaction_history_count(self, read_only=False, **constraints):
|
||||
return self.ledger.get_transaction_history_count(
|
||||
read_only=read_only, wallet=self.wallet, accounts=[self], **constraints
|
||||
)
|
||||
def get_transaction_history_count(self, **constraints):
|
||||
return self.ledger.get_transaction_history_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_claims(self, **constraints):
|
||||
return self.ledger.get_claims(wallet=self.wallet, accounts=[self], **constraints)
|
||||
@@ -615,7 +622,7 @@ class Account:
|
||||
return self.ledger.get_support_count(wallet=self.wallet, accounts=[self], **constraints)
|
||||
|
||||
def get_support_summary(self):
|
||||
return self.ledger.db.get_supports_summary(wallet=self.wallet, accounts=[self])
|
||||
return self.ledger.db.get_supports_summary(account_id=self.id)
|
||||
|
||||
async def release_all_outputs(self):
|
||||
await self.ledger.db.release_all_outputs(self)
|
||||
@@ -1,884 +0,0 @@
|
||||
HASHES = {
|
||||
0: 'bf3ff54138625c56737509f080e7e7f3c55972f0f80e684f8e25d2ad83bedbe2',
|
||||
1000: '4ec1f9aebc8f7f75d5d05430d1512e38598188d56a8b51510c9e47656c4ffad9',
|
||||
2000: '5d5965e43d187b6f8b35b4be51099d9ec6f7b0446dac778f30040b8371b621a2',
|
||||
3000: 'd429f69a9dd890f7d9827232a348fe5120371ed402baf53c383e860081f6706e',
|
||||
4000: 'd18fef650f23032e4e21299b8b71127f53ff6d2114c3eac4592a71f550dc5071',
|
||||
5000: '12c1dd1b9cda29eb4448e619a4099c7bb6612159e1cd9765fbbabeee50f1daff',
|
||||
6000: 'ebda54cab7979bc60f46cab8d36b6fcea31a44c38d6e59bdcb582054b6885cf8',
|
||||
7000: 'd2be08791053336ae9bd322ee5b97e9920764858f173a9e0e8fb54ee5d8bad0e',
|
||||
8000: '94583639d9e2ce5eac78c08ef662190830263ea13468453b0a8492c3aca562fe',
|
||||
9000: 'a6ebb24a3ec9bb3fbdb9d9525729513dd705fe85093c33a07bbfe34ba1f1eae0',
|
||||
10000: 'a5d32c6cd43097f3725e36dd55d08ec04cbe62e40d1fb3b798167cc9c1fa1ce4',
|
||||
11000: 'b5cf8df354bc26e8b0315e8038564a9a7fc1faa6ad75e67819b7eafbcfdcb145',
|
||||
12000: 'ae7b0e5106299e43b78f337fe15e85c27f88d5bcf618a21c03ce43c094dde909',
|
||||
13000: 'b57dfe631571b654e11fb4295aecfc11f612664d22d64a9690f64c0b3ec128bf',
|
||||
14000: '76e55cc1c38d9a6a349c9a42ae0b748d0d8b27ebb30539be8bb92e71e7abca17',
|
||||
15000: '5d33cbc872acb824a89ef601aa3c918e7f4dbb8d5be509a400fa41b03f7fe103',
|
||||
16000: '25760f6a6bfb58104d02e60efa62ba5ae141bbac1f65d944a667ad66fbef15a9',
|
||||
17000: 'ad37d98bb272aee3a950679cdd4cc203b1c129bb3a03fe74223ff858badfcc50',
|
||||
18000: '9eac34fc453b97b1c62228740efcae4ef910b0bea26c322e17e3474b81713b0d',
|
||||
19000: '06783429c8f50cc6495da29d38c86c3873800cf1eceffedfffe2a73f116a39d3',
|
||||
20000: '0237c620f7f68d9ae9852d00551c98a938352d2e3d9336c536f1cbf5246fade6',
|
||||
21000: 'c4dbb4eea7195a2cf73de2d49b32447cec9c063051acbd300b26399c9242b0ed',
|
||||
22000: 'd18fea80b659d9690830c2f37c7c9d53f79fb919debb27cc2131044df26d7ff5',
|
||||
23000: '0371254f4698d3c46596c1fd861e2e1e75053beb69c3c4d8ee6702f0992f0026',
|
||||
24000: 'dbc97c3986bb9bb6230321d1fc8b7ca13369033f57b08ac01bb775f7241861fe',
|
||||
25000: 'cfb9baa125e91d02c060940b5b51cce52b495a0b0bd081cd869a7ca45a0e0585',
|
||||
26000: '6e07fecd3d0f72b8cd50a3cdc1e9f624beb3dac9f6356308a1ec295f47d68f3b',
|
||||
27000: '8ad248da3aac7c1db958d0ce8a2d50fde461bbb96a628efa215116d3b74b74e5',
|
||||
28000: '4e8231ede23b397c49f1a97020ab5fe45465936cb6c8cf1c39ade1b38a80547f',
|
||||
29000: '1453b7a4be4076e2c77b94bc35f639c571dc05b225a5f8018a9a60337fb3ef66',
|
||||
30000: '43d779c83032d75e8df09aeb6ab103ad41618777565fabc2e56bf31c3a20285a',
|
||||
31000: 'ea1426f57f75ee781a6e569772944119f5e65c8b110b31a32b61d951c3a1900f',
|
||||
32000: '43d1a3941b2c3033f78040830d39e657f74958d71b3cd0bc728550469cdb5fde',
|
||||
33000: '3b0c0eac231476efed2b9bb9dff206a6438a14946c3b4238bf0af6d03f9a350f',
|
||||
34000: 'a8136e9d6f3f5048ef8f9f56628009f5a87bb28664ba8028ecae0797e5328309',
|
||||
35000: 'd8136e638995aaea03c6b50f1cc0f1891bd107290f12a6ee57e3cae36b7e3374',
|
||||
36000: '06a7692cf5ce9bb71461e5427e6f55bce2904f9d3af4c28e7343fecbf0e336da',
|
||||
37000: '12a1a49c237497295150f99c76d79d8affe9fd720e0cbffa62be80e2bda7f832',
|
||||
38000: 'fcf3e526931b7f7cead47652f12cdf76641032bc545e22aa0dc83bef46a085fa',
|
||||
39000: 'ffc34a198ab1cc13819c0c90b26a8455627ff12868b49ac1e6b5e1a086ed011f',
|
||||
40000: 'ac71a9895999531350cf77a701035b0b59720a39994c990354b9cb1f6ea8eb49',
|
||||
41000: '4fa753f5f2de41ad69ab32c6802eeb352f070d8684c3bd154eeb8a3c17aa6363',
|
||||
42000: '6f0febbddf9248e3df2254a7196ec13f353c7a8049e098f2fcdf920580abffa4',
|
||||
43000: 'e3e98488cb203be9b531e3b5ce2200f4e5655297a4f4534bcf486ad5e26687bf',
|
||||
44000: 'c7c6078e30163204e49d493d506f72f71d68d2a591874328342ca3cca81c87bd',
|
||||
45000: '2a65629529f69c15a6e404ba54b83e024acba9d6c786be554fb14d28951ddf9b',
|
||||
46000: 'cbc148ad661e6fc390746e94cf21df71edcb222b4e8d8f0771237a8e03412f9c',
|
||||
47000: 'd6977f770749b7ce1a624a61f7b2d26d817a27c50a895466ee10f8bd758c029a',
|
||||
48000: 'df05fddcd5ccfab7811e5d7b6cf7b49df71c7e0f77a57ce384936948114bb93c',
|
||||
49000: '9d26a67c06f066229bcdafd55cbe71e2b2518465d6deb618c418b2f229ef513f',
|
||||
50000: '4ceb896d901315e0f3e10f3b1ed8472f5bb5230d4ed6377214b07e425e515f6a',
|
||||
51000: '218019bd8440a8c242e84940119d4eccccd26134a208cbb92cdea3480069f482',
|
||||
52000: 'c15f1538280e71abc1cf3e117c79b0b671a9f6fb46102fea7b9a1fbcad418011',
|
||||
53000: '0ad4d3812a42e23fc0defe7c62dad451266ee21699ade625a5f00b91993850d3',
|
||||
54000: 'f994cff7b8166b0c39a310bb686f043fe9355678ae406a7fa1a74d4bf6f9b849',
|
||||
55000: '6723d30c99fb45513a42a3ffaa50d1b8350e3c8b52c6dc6894303147eba95d55',
|
||||
56000: '947cbea79aabe2051522d9e957d92b86dee2c420c32a83cf6f6d655136731926',
|
||||
57000: '8a153fa31a971e7e304650257bfdbaac230b36a9beebc01a85a7aab9a357a5f2',
|
||||
58000: 'd19a5234ab995cb851d691b2c2c84ae665890621c52794e54c16d6a87f516c7b',
|
||||
59000: 'f9c7a353e62ce57a5e3f983362b08fdbff5e2aaadce325bba7680104a6f51cb7',
|
||||
60000: '9e1d535bd31525803e93e4b4c2346fcdfee7c5cdc4b9fe3032df9295aa85c754',
|
||||
61000: '640387271e7eb0816b9e290dd157f58d1eb54cfc39bc7ecc4486099af2a22b76',
|
||||
62000: '3d36acc4169a238657729df674277c6cf236e21c2bed4e99b8f1a6e82d00bbd6',
|
||||
63000: '6d4201e45595ea160773d63bb1481f78664156de1f4564d9a8c3f61b3cc1a6c9',
|
||||
64000: 'b2cadf2ba39b2d5e688029cbbc68cc6d71337b29861dee24fe941b31be1198a3',
|
||||
65000: 'd8a364d385ef42e53ce88c82355235ee7a27dd3e642849dc22d08a4041695dfb',
|
||||
66000: '7025d4b2d9537002ce2b9ec42c596eb864e30f41c06f02413997b5629d6dce3b',
|
||||
67000: '852a3fecfb3a77bcd6d2604170cbe9646d9b889c90d234eddc8f9d112777fe41',
|
||||
68000: '38956213012f66a89377f4914db838faa6738732f7a7a865ebc5777f2f126b73',
|
||||
69000: '4c89b07d23727d4bd1a080bcb9a22dcdb45c904f4c9866153a000056d7ec1546',
|
||||
70000: 'e4e7fbf0d17b5f933deba12e045ea23eabfa6f7dbdc633e5c3d3ef8ddd066b47',
|
||||
71000: '1f07c36e82f8e7582ec9df23ddc90715df45978b6aa0a2f17c08ee14cbb23000',
|
||||
72000: 'c59c660b5aca2fa3364524ef43a4fa93fc3f16df777427cf1d810f65ce42e4d2',
|
||||
73000: '83a05ac2fcae8c5d50a90df308f641616fa9b9fb553763c874b764a73def2f95',
|
||||
74000: '448d209c032a2c6f5330481bf34fcf0bf1ea2c47f43fa5f82cf42e0d8f4d2b20',
|
||||
75000: '5627eb3491e1154ad2c90094da85dc7dbfaf89d3765848db1ed915b7e26dde58',
|
||||
76000: 'b1417b1b2360e24fe42345508ba0a333f4bd76f48c959b8bb40a99bf34ec5ba0',
|
||||
77000: '250aca2bc4e6da7a8e1cb2e4deb0396aee7c317177b099d7dab9731b4c0f7573',
|
||||
78000: 'dbfb1dbe80a5db06c9dcc645c5279eaf40559f5e04df0e64b696e80aba5659e0',
|
||||
79000: 'a7183a545925164b5a9882810ae117d0f3df9b3ed55885fe74ebea284bec484f',
|
||||
80000: '5b4a7bfd9b5394e2759daecfe472ea6e92b281a7821b106314b2b2e0facbaab8',
|
||||
81000: 'b7075fe530b44e4f5895521b3cf99a79fe23d8bedde0027d6e7924ca93793eab',
|
||||
82000: '6a1fa75b896d799ae84046cba7cd942dad47ec04bf50a85cd1b2bb15861995e1',
|
||||
83000: 'aa5f4caf970433e90e2e45f9524ab9a6e22281505ada4c206ecf4672486240e3',
|
||||
84000: '103732d12ef792a1adbe8f295ca6abd008867323ee996698b2650bdc1bb8d06b',
|
||||
85000: 'a214c16ffdc505d6caba7bd0c2bae766bb19f7eb4d436cfc037e93783afae7e8',
|
||||
86000: 'f54226ae9a6814a345968b5c2982adf638b995ff50e27de1890b3760ead12158',
|
||||
87000: '26546212a0720cab988194432f4fa7c3e47caf0fb8e31efdd1ee93c3ad056868',
|
||||
88000: '48f7d47bc443bdbc6a37ffe8f1d0d91de16d4a470d03655692de8a04f84c2561',
|
||||
89000: '005bac455ff093a39b907af0441897e7a9ceb19eaf51d9dd9ec2b5ea43ff6d57',
|
||||
90000: 'b482f10315170e6bd280325e180ce37b0423b685baa06d33586ee659649b71a7',
|
||||
91000: '56db4d5d6c460fa76d492d9900c02e8fa99bfe97c7d9fa2a75bf115128ea0da7',
|
||||
92000: '72ac917c4f531f91f65fbe33e78b4d4e2bef18aab3d2ff584d59fed1ba2cf398',
|
||||
93000: '1100df2fdb03d44530723d131dd7d687053db6a317ac9181cfdd51925152df02',
|
||||
94000: '6d50f66efb571136501340fc00b52105e0011f0f11fd68c9b185717bd42f9307',
|
||||
95000: 'bd0c996d9ec40d5e91c2b5f278a5c9c9ea0b839012a7d135e1afc73a725bd8d6',
|
||||
96000: '4ebfe8a3e1a1625632896d5d2970d8b080f08c5948f0ac38e81f3fc85db5af5e',
|
||||
97000: 'cbc03464aa3513aa33b540d0f026b93db7607c4bc245af8fd88c3b476ea5e394',
|
||||
98000: 'c9a185ffa55d01fc73c2e60004b0c8dee60ffbac7784cd16046effd7a0a51a86',
|
||||
99000: '1eba97d5ee69229f4cd697c934d60cf3262ebcdf4c79526cda71e07b52fa22f7',
|
||||
100000: '0b1650207a55c64e7a6cec62ddc3cc190ed3796507bbd48358a0b7d6186bc3ce',
|
||||
101000: '1f47841b0034d3cf1046660c2246902f679445b5ef621df2762bbea33b7d685a',
|
||||
102000: '42f63b1b622d08e120b5405ba65d25e1a9777312cca77248b7827d7084e6d482',
|
||||
103000: 'ef4861fbd4e578ff80e0d5e2afc62fb113aa203d3933f74efd3e2400d73de922',
|
||||
104000: 'e97a3cb78c09eef732757e81c6a7135beb33d95398054b45df738853acedaaad',
|
||||
105000: 'd12e8feec15893ccab662a1ad0754b2ccbc18b078ff3894eb2214ee4ac2f7af9',
|
||||
106000: '83ae84103b37c93b2d1535fc47b053194270b3c186b1b25d69cd8a1540caaa1a',
|
||||
107000: 'a102c6dce88bc4695250c24a0ebd2913b2b85c211b7d5dbbd18ecd95bea63144',
|
||||
108000: 'a0c20b36140d55288dc4dfeb6c16a6ece3d43efb57b1728bc872c35d6660c704',
|
||||
109000: '412fe428b2929c3cf5c6991a2657e5414a569b30e77a3f4fecef14239d89caca',
|
||||
110000: '8518706e957f3ef892d1d2d7f65c5f4588620be994cda9cb7d81fb600554a456',
|
||||
111000: '66f4e25f40d24360299e5358f954d8a79b245399c46dbc4e8a3db0c10fa14e18',
|
||||
112000: '81c0ded5c5e30f92eec84ca293abb0165068d917bdcf436754a0e9b0ef1e325b',
|
||||
113000: 'ebd5e034a45517e59e96e986b55e38051a5d2a3120a7ed92d4f3e01e18e73972',
|
||||
114000: '681b85720d71ab660c7443392883ed4bdcf9aceffe202cc5ead94664e315a744',
|
||||
115000: '3fbda839115bc6e6d451a05f78b27b35b137c94374fdaa42235f3035980e85e5',
|
||||
116000: '7bfb88d39fe0ee7ce046648676b226997a812af1f1ed79040750045536c76067',
|
||||
117000: '5853c6d48fbf99b06d3e1f474e1b667ca598712f9feaff8cda48a95935cfc498',
|
||||
118000: '6a23987731379799289d2a527dd40fde8aaca8398625d2a2e3e646372ffc99e1',
|
||||
119000: 'af73cbe957865b60c3c3139dd1e2bf8bf1cc504dd167545463105996609750c4',
|
||||
120000: 'b309a20170421fc4ddfedfd7611716e1cf5a802a1db09b66b9ae448fdf958792',
|
||||
121000: '624cc893801e4da093ca973e2135352bb428dc278e47f5308412d05acbbdeacc',
|
||||
122000: '2f20c67ac3485b6e5407762a23e8a7df1bb380f1607f538cb5e2e1ffaad578ef',
|
||||
123000: '89c73ed65ab8f729cdcd678dc52e876969907f1560898220702a696893744e00',
|
||||
124000: '17d8155d97fa4e8705aa55969709f9cbc97f87815d40f5a2b5cf4d7b85dfe4b3',
|
||||
125000: 'd3cbbdb7f51c8a1f5794bb3d1f40c413b313f91bd3f3dec0d8ebb6a02107e52a',
|
||||
126000: 'bfd02d760eb5de2e792ea74215ae37d13174c3fc0aff08587281f0f39e427caf',
|
||||
127000: '937dcc09a14d1de79f5eb679996225995cdce69497cff5f12c39c098c42656a0',
|
||||
128000: '81d372b07e2467bfa62bb4b974d0eb75ea313e547b9d4e62f3c617f939bcf67d',
|
||||
129000: '9b4b315738ab8b5266b5accada140dcf487b2f509f1d758b9b972b93ecd03c9e',
|
||||
130000: '8ab3b1862cacf7f1c4f6617f7280ba31547dadd62bec9c37ecc8d074c90be2b0',
|
||||
131000: '868d6520e526b8594c44a05eba67ec1de8e01b547b34b62ebc5dd09d656e9072',
|
||||
132000: 'ce09471822965da31db3188cd156538c44337da2952486a34dd2ab372ea490a8',
|
||||
133000: 'b0c34cd700630c9281c8657dd24baf609f72f897442538d77f4ac355199731c0',
|
||||
134000: 'a76e5dc895406a3951d55b557272915810420bdc3ac076863c7efdef3b115890',
|
||||
135000: '2b70fd7021d40f6df15f3f72bc8e034a8a0bcd8542847b5b7a2088253351b3a4',
|
||||
136000: '6161ee84f6fe100a02de0675bc9500dc18ccaec74e7da4b92135f0ccee6fb663',
|
||||
137000: 'ae5770a7158089ca9adab8bdd07f1259e6f204ed377af27fb6a772fe1c031864',
|
||||
138000: 'f3b8de21370968a90cedfcb1cb6803ec9a6b7740a94d8e4c60a80407fd55a12e',
|
||||
139000: 'f08f6431d7367b71b25b321809ebf48f797bf7fcc943ba366420fd3f3dc00e5b',
|
||||
140000: 'e76345dbb8c4c5ed1ea37249f892f64098557700259fb360401a559c45909041',
|
||||
141000: '776b333f5b221f6b443d6011bc8d0753eed5ce2446a6d3ee99a5800c7159fa92',
|
||||
142000: '9453693cb846f27ba2ac39194e5b70cedfaf435b73b5f7f288dd81343ff605ea',
|
||||
143000: '08aea64cc4eb0170d2704cbd3a4ee871d4f3e53a8c1395288c029eaddd1aa097',
|
||||
144000: '801d60c225301481c761114d5152780d5002b77aed18845c32c9e3804a5db254',
|
||||
145000: '78cc80d54933b84248aee32a18144d03237f948de7889c97e2112d17ac998009',
|
||||
146000: '1e954e8f2fe59da3feef0514f6b271f52961467782b2530501c294a83181f176',
|
||||
147000: 'c0ac42fe9c0e1d2b0b7e6e796c553eafbd5d7bbce7df84903dabe16dc56ba863',
|
||||
148000: '75b5441fc1785dcba69b9409f24224ba3207d624bb0e7d54af24118d08c788d7',
|
||||
149000: '20e53048515cc34507cf2d1dd160eabafaa97793d81626e9fce38d540142dbc4',
|
||||
150000: '5ff7a08461fc3fbacf8c6261dbd34a790e271bbc39908c23d5417f0f7a71b71c',
|
||||
151000: '5c14cab7670821966d5aa61f09b32d93c865764b6e258764a3c33a82a3133fe7',
|
||||
152000: 'e595968e040b54ed6572862b4089e005238399014d99bf4c361cc5922dbfa6c0',
|
||||
153000: '1292bac3fab11de09f46a8327511b7c37366d0f2a96a027f4bf2f16147a39af3',
|
||||
154000: '3db31267722b500dd9b26f5cfc4f06f26c5c343d181464499d3bb4aaf13c3c20',
|
||||
155000: 'b92962af73b42335125f28d7d3fb97e3aa6ccf47c3f4229eb7fa04b67fdb933e',
|
||||
156000: 'fdbcf12415284d8cc0adbf16996b9d37a943fd1a924c5dee5f47dc8add23f278',
|
||||
157000: '199781ca8e5929708f834d6f38ce3e2fe196ab519f9915f14401462273431e19',
|
||||
158000: '969017fd15cf6a73683de287c5e19fbd69ebb5aaf7c13339297f7177a716de3d',
|
||||
159000: '77c7995cf97218655c195fdac9010c599859a46f760d84750189114d2d2d1d9d',
|
||||
160000: '00ff84f0d845b4f1099d970cc0c3e2dd1c2584c611449f2318a3f27327246d51',
|
||||
161000: '08481dc5f61e776ae8be12236d595549abfa0be28b187d80dad573594d94c11f',
|
||||
162000: '71497ab26b05453f3c7057f8bf57fcc8ba30920a6032ab0ae3abdd29e0677582',
|
||||
163000: '08713e0b750233a3843241d24573c4800f32c894000b7815860048ef3e7a06db',
|
||||
164000: '01fd806f1879285ad5234f38789074b0ea3a0b2707bc6b49aa9bd51ecd26ddcb',
|
||||
165000: 'a990225a2f02a77c1c61d518d46dde1e1cff3f4df0ed24eb463b97f84c7a2616',
|
||||
166000: '7772a601a6e765ee340eb666cd645503c9988430ba4f558961246e9e69349b25',
|
||||
167000: '389c7a8979078b57d54936497680c7908b9f989e46631358f1c31a3094621557',
|
||||
168000: '2e1df97fe3cba7add5f05734a119464ff7c22bf92701ed85642da856ec653aff',
|
||||
169000: 'fe0adfe455f65cf90e1632ef39bf9ca5856020d995cb71ce8a4998a40eed5998',
|
||||
170000: '87487f255873e9f6411565553fa9ccb9518e7baf71ade67492536278f1ba0feb',
|
||||
171000: '37cf239bd630b7c891019adcefead4baf19a86b40e21f3dfe4a76a78f2985103',
|
||||
172000: 'e2f4087b868558af51dc8f40d77c300d76eacab17e5fb8bbf3b1f99c02d995a7',
|
||||
173000: '273f32717f740438e93063d259dd4f0f160077c34c937a7d6a9b1f9fbb34d87d',
|
||||
174000: '807a861813ca690a6fc0715f354f36bf491c34588b5ab778c0f4e692ec3fc152',
|
||||
175000: 'dd74cf3d686c59820885bab134c81b7079f2edca86a25944fe0aa36a96941550',
|
||||
176000: 'aba8163b91a905aab5ad6e7599b919fba650c446938dc7b2352759203bea8957',
|
||||
177000: '260bd0d053038a54565f9b25c3894cfa875c9426b9d6e9915ec1df03bcf90ac5',
|
||||
178000: '7a527f71743c4dff25f13b918ad4a6d91fb0bf9c873a7282e51cb4edf545edbb',
|
||||
179000: '945397818b80995ac4e23c785b6d5ed2eb01338a2d4ca6f0cf40986e87659d86',
|
||||
180000: 'ba1b5263f5418844796a19db06a53c095ec0428696d8ee953790d6c86de0795a',
|
||||
181000: '2b673859af43554accd8599bca53c2ca94500ecd65df7672a1e586595bdec7ff',
|
||||
182000: '08bae9f096e348c8c6b42ac2762298ec3faeba896524968478fff1d40017b5b0',
|
||||
183000: '211577ae9a4f93fa5dc3e33764c31048ee306b5e5ffa5081fabdf3f30e49a977',
|
||||
184000: 'a4301eead637043a8751a30c13935af971a731e03e90317c5a16c1677e50d837',
|
||||
185000: '1f4aa362281293e8328b6c5b32a9948ef03dcea81c6ba13f94531f0c5e42627a',
|
||||
186000: '0c6b15f2164a1b55bdf0eee2b0f5ab7cb9d2dfea6fcc82686483aa7a3659f6a4',
|
||||
187000: '5bfa261cad0e4d271814de1f4752352f35da10f24b8016ac0065172e1296052a',
|
||||
188000: '94c2490bdafe70d65951e4034e5b9122fa6b376a5ad8c9e6276289c4a5e059e0',
|
||||
189000: 'a0119c5d57523b7ccaf1e5c6671eae9f8562bbf2477f7dd8c6847c6da41707f6',
|
||||
190000: '4d82802c2464053c391374eb6124f8839faa9b343bae3f34e00f87fad9f45f9b',
|
||||
191000: 'd341f5b4d20894be520eeb533d56c5737ad02a7ab0c24b2463516ff04e3b8b0d',
|
||||
192000: 'ddf1a72ba512786d39f7c313f710bbd452040f1eb8a0c0ee7a4b9192199fcc3c',
|
||||
193000: '514e1b40e5e15dc72e75cabb8d490c5e76b968e6dcdb3acf518b47e3a2020407',
|
||||
194000: 'd0bac7ca0636b662d6be169bb5af807c891523a44fceda68f08c8934d6e9e254',
|
||||
195000: '00c37e81f0dfe63a8fdefd0ba91442019899d50f72b43c4de8017e767777a5af',
|
||||
196000: '581a9d093458c0157464c42fa2e2d9a1f61f7edf1ccc5dd77a4fdfdd9a4b37b8',
|
||||
197000: 'd7555fbf7671d2475e38f62418f6130ddfe85b8fd07fe8a9815849316165d401',
|
||||
198000: 'ea388884ad5abb06ccb1d0d2202dfef5c4ce4de0f3040df089017d01f14b9530',
|
||||
199000: '6a80cbb22620b7a5f99f4ea36380b7bcd22e87feaf4186bfba0b9aabcbcaab70',
|
||||
200000: '7a7231293d242887c6dfa517a78b60d40b41456b6f2fe833c0217074664a61d7',
|
||||
201000: '0d6bde8cdd0370964b83ee766891b3cda7b9eec098dfaa5ef05842b29bbd043c',
|
||||
202000: '223fc091bddfbaedec92099f873fc1a6bc50556f3421f472a63977b3351b95c1',
|
||||
203000: 'f87fb207fd397f8fb5c5c21f84f272cf33d840e20ccaf9de810af784d8459142',
|
||||
204000: '6273b130d819d84492bc50fa443ca401df748e58fa84a3708a1797f116f60f1f',
|
||||
205000: '498edfaaa9d5f0501aea5482ffdd9208b86af46fcea5a1cbc82d51d45eb6a256',
|
||||
206000: 'c069138610493a70742918b5c624cbf9275bf55ee24649e32244238433883ea0',
|
||||
207000: '5110744882ba11b98df95f11e100bc398a5aab4c8258133b518b5abb58c7fe64',
|
||||
208000: 'e5690d4e0561ef015d6e2997df3b8b95163cd884d999d8c9ac212bd7e1f06ab0',
|
||||
209000: '17a0a3bbf2fecbfa5c7667524cb2a452f49b8dd089af132f9d2c530ca0b677b1',
|
||||
210000: '3a9489f3aca89576152949a67df6462d25bf3a78976451b5cf8bd8385a10e6ee',
|
||||
211000: '45196e756c7c282c4a9e7133adb07f103d0f74ee90919cc8ad89ca3e5d38c6db',
|
||||
212000: '590f21c83694c35028b9c9d632a082307b8bb27ec87e2c0c4e2881a7be9c6637',
|
||||
213000: '46e56c45305c51e4d22927e449a1855064504cfb58cfceb0f293d8a4f1dca7cb',
|
||||
214000: '50b3f1bb82c1b213ed2cfd2c1144b8e67e5fa731dafabfefe063f8658c5536d7',
|
||||
215000: 'f53709db4ca6c12b80c64501eeb8f176211116b0fbc3a650082cf830792b206a',
|
||||
216000: '6818db3f6582e71021a84fffe751fcb47db2f734d59547be47a373575b5c7d8e',
|
||||
217000: '01f6f8a0699ac533ee7921f63334af07b04369d5e580c1c609c6fee64c8c6b8d',
|
||||
218000: '81e9ff93cdf5686fc42663a056c74fb0f560c29a5d23f21ef92364735a865392',
|
||||
219000: '80b32bb6bc3c54a19d3b604f56084b3bda13afd4af4130f791505e44fdad6c98',
|
||||
220000: 'c427f24ab14449b1430c5302385f3a77f90f19715d940e8dfb7c054828270163',
|
||||
221000: '4303a0e1c6f8489589290370a6843d97eb02a01be8c227d905cd0e30034feae6',
|
||||
222000: 'ccfdc1768afc18cccdf2b5d90d40158d86b110d878955b6570ad9e6e40063008',
|
||||
223000: '6baefb7061239c02472c4783de6d32936f7fa1da9fa6d2448b4d92f40b89e79d',
|
||||
224000: 'bdc334f5d1a3bd1d19cf589e3b1ab0e8a324d44ea1c70297ff2de6a66c99399f',
|
||||
225000: '82d29612f96c55dbe83655db68afe39146aff1418af929f05033fcbe612ebcb0',
|
||||
226000: '179b4144f4d644f546f001273c1e5382a8de50f84ec06a2c83b02808a9167d28',
|
||||
227000: 'a350e96895efe979223475151305eb8d1739121b566964033b9967031ee5581c',
|
||||
228000: 'd208c66a9ccb1afde8d1540d944bfb43522015f9d2df50851e66ce1b84b06c0d',
|
||||
229000: '95cfabe5d18dd8d8e4a57b7bf7052ab09b7138ba0b3c9d2efdb32352bfcca149',
|
||||
230000: '3e399d08c33b8fddeb687deb276afd3ce101d19f5a71b232532951f05d46d8f6',
|
||||
231000: '91c0c17d5df1c553bd7dd6db8feb58a4436d598ef4acb1ed9a06da65f3cca83f',
|
||||
232000: 'eb3928a522dddde176cd4c68fa95f1e953d74f3759af4b030f7e2be4c466c45f',
|
||||
233000: '3ac946df5a319d31716f50c752a5769471c0f3ba4b1002613996981c8f32fa3b',
|
||||
234000: '4c9cdbce1f2a4324368130c154e2d24e885877c13a126a847059ccc289fc922c',
|
||||
235000: '8701d98b254fd0b588f022ea140e936d496ca8ccc0890ce133523432f4c09e6a',
|
||||
236000: '0d2d45adcdf950541c25ff5b2b87dff5faad479094b1f68804a62c3151e6d598',
|
||||
237000: 'b8924ead720d0e7a304c45c209ef3fdb90cde437d138bfed2f1e8357fb3d951d',
|
||||
238000: '4c9c62a3b3f6bbfb08edc23d63f9b35ed94bb38393fd1761f9b9b810bb72f68d',
|
||||
239000: 'd01494b4999c2c4def2564eeb1207fd1f3d35a62fba2e61b386bfab498cb9b6f',
|
||||
240000: '3021d8f0edbc0cd2ab87bad5feffac54924b51eac9f6fae8ebe289f474f11a8c',
|
||||
241000: 'd2ecf0170093d3a2dfc876f673e565e3e4aa76d23896af51c96c004cb14ce67a',
|
||||
242000: '2b5ef67326178702a1a35fe4a848d22b791114b59121e41a25ddc1bfac82c6bd',
|
||||
243000: 'b2d546f127252c3fa5f7ca45c61268dbaef66027484a1bcdadd281fabf97d34f',
|
||||
244000: '58f9b398b871dd62952899d31661c118ca4f0d1c71dd165523b889d62b154393',
|
||||
245000: '504760f0e46e2de0a2c8b4dd3e93c19d0f3e0e1b36916f02cbff166655816206',
|
||||
246000: '15ffb23b844e6ac648a7fd099be8e631cb51b3a2496c67ac2dbedc4f6818d7a7',
|
||||
247000: 'e3e56e8575552202cf35fc883bfcfd92cacf8013a6536729ec19e53bdb0db128',
|
||||
248000: '22f75d137c7fe42a80ee04a386fdea8cad56e7ce8601234b6ec676498edf9e34',
|
||||
249000: 'f84709269c393cae8106ddb1fa3a48cf3226e97bcc108717e483493159921aa1',
|
||||
250000: 'f3aee545330f18ec49f6f880d7f9fd31ed6633279881dd13177b0a687c51b000',
|
||||
251000: 'c2e6a4cb3c6e88f523db2ac24fc7fff3b8a27d13ea7b5d614806ed96247025e4',
|
||||
252000: 'efba2c1f91a13ee8d7a8658e70893ccd3014a7019c225e356c3ee67955b58756',
|
||||
253000: '9097ecbbc1279bfa2c92822a6485714ca7ba35cd3dad9ccaff5a8d24645eae92',
|
||||
254000: '00a41b720ec0d7a9fea3229b16ad8d065cd6bb1d14587eb3404de33e45db9454',
|
||||
255000: '7523636406176d391928c638e15e959a9d0255aa71dc9f3bea8995cae02d1d51',
|
||||
256000: '8d3f7c5ded24e2488e25d6030d78d351354734b8b058caf6143430e96ebd5d0c',
|
||||
257000: '48fbbc3737a718163a501e52964eb58aaf4163aab7ceaa4f75f928b25c376ec2',
|
||||
258000: 'e1052e7ebf83246770a740c0c6ce8f011fc7007448777c533d67fec810fe5e64',
|
||||
259000: '043f7d98b500fafc248549c1f5a9727fb0cec4e3e5f13071eeba459fb9179d13',
|
||||
260000: 'ed2f3960b077651ac554b277dd271e12e6cd05942fb54b9464f18432adbe5ca2',
|
||||
261000: '234f17bf2054ad4d3efeae165c3359a14e08d2e53506c839cbed6ba3d2a48ab5',
|
||||
262000: '4342d662c4dab581bf1c123e057e4413bbf74b7a2a331e80f7099201539be54c',
|
||||
263000: 'a7b1380371fbfe64bfb94bed5ae256f7db59a4661e461221cae99dcd54c72a8a',
|
||||
264000: '4f2a606cdc5f346e0855d962cbef7c2e4993b2b07b86ce658a796b1baf6179d8',
|
||||
265000: 'cc09cde8c101eb17b5457bf976525940904cb839e70ef3335780eb0261976826',
|
||||
266000: '3c5f24533723afb124b2bc8c2f8d373eb5640dc4ea4dc76b841e81706dd6b29d',
|
||||
267000: '467c5ca63621be011a07129d0c3ea785e2a0853cc3c0d1e9cace122831d42507',
|
||||
268000: '07df13a794f6c0fc77142fbd0afa23121ee2a2a23bc9abca35c164edb5f6ad1c',
|
||||
269000: 'e2be277e83a997a111fb45948e9a7afdd7925feb1708a84c62968cf6a42da1c1',
|
||||
270000: '463425dddeeac2f27ee717364073229c333a7ba357888623c3b15a1db52542f4',
|
||||
271000: 'd62e0bd8584fae7e4893c18252ae8d1c245e2e4c8e2f2d73703c681c6259f19f',
|
||||
272000: '85cabf5254b6b6292c105e68d44f78d6effeec582467caf9d30a5deeb43b6a6a',
|
||||
273000: 'f5509cee847a757c84adea8e40f3c7c0cc6a4da33a8cd3019cf9ca850d0bc93b',
|
||||
274000: '2cfdb3477de2fe1c81c7133094094c79e933cfe6034c158de07ffd0c4c08b564',
|
||||
275000: '1c1b8e447e95ec3c306fc68528631682bd85e1c876d86223fea28b7ff383c16a',
|
||||
276000: 'b37e5e4f6d29590c9a58773a4d4384cf7c51d04fede4a602ba8095e14bf65993',
|
||||
277000: 'b92d6ffdecdca014c946c2fe8c6c404ccf60d8401e7631918da8885bd360daca',
|
||||
278000: 'c1f82e49c8f66aa580a216c8e08262080ff9fc4f4df1c482ce8b2695b90b7240',
|
||||
279000: '41c48f5965e8939d1df69127798b46af17646ea75e545a7b47049596d9bfb3a2',
|
||||
280000: '48da55fb8ce259432417ffacba5e064df4df2a8293cdd484ee0ad8ce4b0bc38f',
|
||||
281000: '0e892fd450a8ca018567e1a6e23bcdc5b4b872686f816a415991f59b3abd76f3',
|
||||
282000: 'b90fb78edf920d0bfb4bce1f39327c6777b511fd05d715729d7c4cdbf449d59e',
|
||||
283000: '80965fd3817b9509ddbe6a919aa73adb11af64cda5d8352ad2bd9b7824e95c42',
|
||||
284000: '43447043aea93b162d6f615e8cb7d992d02c176a41b0518256f7e0d8fb3162b1',
|
||||
285000: '33bc92c77ba88251822698a7bd02860ef9c12420e4c27f4020370acaab0a2f37',
|
||||
286000: 'eb664123736754c19a47e2afbbc8645c598d7b70c3d42132c334cfeea8fd6bf3',
|
||||
287000: '6af8f2fb3ea1f0fd0d6432599c3a681be7d384cdc553903bcaac8de2d472725f',
|
||||
288000: 'b5fcfef1b5f6373bfc9322164160fe98fc7fb4f3117bf91c8fe61324db62723a',
|
||||
289000: 'f6f1c6f8a2a2905b2ca18845b8bc771bad6e9cc32fd2c8e20a565cbd7781bac9',
|
||||
290000: 'cded625fb7b923d7471ee28276b6983bb4fb4f6281ded9a8637b6974710feab5',
|
||||
291000: '49af69a4855a36d00cfe2408f7d090fb5001913aa06615eca5af78ecb92b1fb9',
|
||||
292000: '2dfe91f7f1870539e6b51ce58f9cd6415ea8fd50dad376d74941a9ff1bf0f8f4',
|
||||
293000: 'b84f14b784cf6eca78d13f4b0db4714be5bcfa59b73974b7106a0206a8993767',
|
||||
294000: '07e95fc3c141dc64ae192cd233f97ac16b1adc9dfbbd685475bf693930b6f604',
|
||||
295000: 'c39cca37abe0b8839f8e797287385c73031496c632a064a89b9dc0730a3fffd2',
|
||||
296000: 'e69afa519ac761328358f5b3ac8b4395d5aa4b955bf1b4c5f3bc0af49cbc3756',
|
||||
297000: '20c93e2d1d9bce1c861eb16febe50e534acc3e2d142cd2d0767bfe076545d1f8',
|
||||
298000: '75970b654ac884cbd1328b68b59963bfa5babacca4c7d48368a1d938feb8af6c',
|
||||
299000: '05448c7f1230870e814858ac9a7858fc72a045dd9e0f1f14b8862d2fdd524b07',
|
||||
300000: 'e254223f85c3ba65a0295a5538387c08ae94e6070f5c989d27349978fef077c7',
|
||||
301000: '4a8bd2357b4ce05422d51e052018fb9f75ce3c07c53b5d1623ac19afff388e63',
|
||||
302000: '18f4a960b810034f5ea02458bbcb9bf3d38d681555be4e40d132f867444306c8',
|
||||
303000: '667cd927ba1e7ab162c17a711bb0ba65f838e543f46b093c86c14e39988da156',
|
||||
304000: '55280e92a3c3b9ab9e1032490aff10c1a2fe9e96cf4f88e49b7a756c0491701e',
|
||||
305000: '8af5b503f1819b23da7b3663314a15b10e39e072377d1b08518a373c85515c1a',
|
||||
306000: '439796b2b4add7dd0fbe14bb3d04f3917e92de2afa07da2a384b389ad9228888',
|
||||
307000: '07fd4cbacddb504e23880dc454fb70659386afa21dab3e862cbe7688218a43fb',
|
||||
308000: '2734a4205be5f2c8ce5850bee8d4ad0829c208911bf4ce833349a43af2db8fe1',
|
||||
309000: '261b84349406a913a4cb700fd218112920910c1c57922e289114c7b903fe2d9f',
|
||||
310000: 'bc652e239edd3476df7bd1e0563595145fd4f2b37dc2ac3bc28ec089a7550812',
|
||||
311000: 'fd7c1b32015a2418efc6e1a73470ea3205de05662c767b0d4198fcaf804c3b5a',
|
||||
312000: '4f017d27b81da7eaf62760cb7fe6613a6ec28fdc18859b6e727b216240a86315',
|
||||
313000: '2d09525e2364e7cd7fe45191fee11f1ce16a2fbd16b56e41f90a3c64ce2f944e',
|
||||
314000: 'c05770490a47280c5b318709c264b2ab951a5bf8c4575cbf1b1e3dbf35323f45',
|
||||
315000: 'ac5da2f5abcee0f92f29479c6355ddafd1f3b4b3cca6659117c17c9fc10fc743',
|
||||
316000: '4496d1a0e4c22a2521595bd4fae60abdb9404f79135311606a9e6713be373271',
|
||||
317000: '40ba7c87490677af9e8b033a19ee5ffe51ba192f5912f491738bb6485cfb76d4',
|
||||
318000: 'f42045295ce30bd4ba9fd9c845bc1228949b4152517185d3dd0598d8bae38481',
|
||||
319000: 'f488695e46b53e4734f4eb9fced93f067f037af7099d0f9510856a898a0c438d',
|
||||
320000: '83e42370911c70bfa2ad89a82ecd02bbc9cb13d24c2a900a0030771898ed618b',
|
||||
321000: '74eabfbb64eb89488b59393ed529b4ee97e0f88a7fc14f798f22715bcee3fd19',
|
||||
322000: '54f42438cf96fe8efa0180a47350002b34ccfe82123a85a9a6dc6f8b0894f0a2',
|
||||
323000: '5509357b42476732435a1d5a6a850e27f62cb3ca81219a94f08dded06dfcbbf8',
|
||||
324000: '1cd477361dc4c72b4f157469b6f3616e060f5d058b44ba864225210432b35540',
|
||||
325000: '060ae9c541eee90a80a11fe300583e47c0ad4d12093d0651310cc6c87ecf2b75',
|
||||
326000: '13ce59868c947ecf2d6aa986501c9f8411a59b4e0b8b6113f4f45dff4074fe59',
|
||||
327000: '16a1532ff983f5cdab6c2d465d58f0369665a926c9d877212d95f9387e8732ca',
|
||||
328000: 'c9608a16fc43e7e94f04b3309c067b0a4f68a66705009dabb39898f41fdc862e',
|
||||
329000: '029b02acbe0fc8610894f0b89d6bd274362195a443fb9e5e4d4485be7786d461',
|
||||
330000: '01181b9529da87eb39510619285c08b05cfa2ca2c7a2fb3ecadba474c1db3aa5',
|
||||
331000: '2371752c35da119184e53dcb593a193fb1c81d6edcff921599328a6142a57e3d',
|
||||
332000: 'fc025a8f319725faa9903eca86935116a2aad8fec6193055d5d5f0f431ba84b2',
|
||||
333000: 'b602651afd9988075887a3f48455d977357032185b7d482262efff9cc97a45a6',
|
||||
334000: 'a9a2bfb6b0f8db298b5ba732f4a23242fa89e1c5038916ccfba3751f7dd1b4a0',
|
||||
335000: 'eeb3e35b6ea94119bf058e6b02a2ec2a706089bc1c360ca28d1b4e2ce1b547ac',
|
||||
336000: 'e834f7ac6bd8e5b46ecebda4997ae73bccf353fa024fbaba7ad4d1d70d08d14b',
|
||||
337000: '229a7017b6d0ffa08374e297b07f12620bcdbaaf5977dd39f3f207ecb03c7adc',
|
||||
338000: '83d86500e8b2ef46a0f1505e29f7401b123ea5ed5feacb49864ab1de284eb574',
|
||||
339000: 'b08e9dcef3e38d631414b4d3a8d8b3dfa8a07d9e5829c515e7db2fec64c60827',
|
||||
340000: 'aa36ce6494df6905f4516530ddca77d88341c0202bf052347a86e090c03ab0ad',
|
||||
341000: '527b50125d5479117ca8bfe84fd5507b2312360908d4cafe8588d40c4ef5622d',
|
||||
342000: 'd1397e8b0f847b6d46c409659e9e10f3182ad9bdbca563cc2598af37cdc080bf',
|
||||
343000: '896576aef0e0dbdb2df8291f2832716029f69718ac79f355425d475a1b2b7b4a',
|
||||
344000: 'cf4b3c75279b0f7d71cbba934228709ff03e3254fdd83a47fde66e83ff393eb3',
|
||||
345000: 'e68c760edf370e758e815d9363ada456fa2568b06abf8626f28a32cc62944770',
|
||||
346000: 'ab19fe52e780e45a716ed97e93084bb3f7916e0a4479f131c7f5f7a2e7d69d9c',
|
||||
347000: 'bc77d338a10fd71daa6c68fa9df2fdccf44b6d9b1d7c6456bb281bf8547b622b',
|
||||
348000: 'f84469ebb26540cb3de0ae03f1f47452f9a9bc6b7195c3b5260d48623507ed86',
|
||||
349000: '5bafc3e6dbe4bf88257c4381cb4f2fea0a4cde1f272e1f1006600b507cb0f670',
|
||||
350000: 'e53ff96761b9c98bed635457acd1856a2ffe65b84e339cea94ad30f13a31e5e9',
|
||||
351000: '5e0eb50d32e90e5ca73639313e5d3a850a113fbe60ac494ca6cdb799e9604d7e',
|
||||
352000: '1a1a04aaef110493bf8fbdb8d1784a7706c6e303773ad0bad6511dd3c1da6c54',
|
||||
353000: 'd72f3b615e244584f131f2f8e7154534aa9d2d9c4a710bb4de44de556b5d1232',
|
||||
354000: 'e81b823c64229c555a50e90a4c3c4654b6673cbff4c62c6e897d09aac00d95f8',
|
||||
355000: '11eed14cac2024588b389ddcd54315923ffff03d6c4c5f32657ba551fd308516',
|
||||
356000: '465f28c63e564296f75a4a22c3ad16714cda29d7a2797e51e512eb1ed5f6a49e',
|
||||
357000: '2b544c79c6f74ae1c05c268669e57ad41d6807f1d0ce1a5024b7b08646df6861',
|
||||
358000: '8a76eccf113819aad8d7c17a459a614274bfdaeacfb4659d945f13dbf3907ccc',
|
||||
359000: 'e63743f458f1479593d73952bfb882c72e5889331add576a716761eff2cf13dd',
|
||||
360000: '0ab0bef439debf08dc3e1df9010c4e119d6862de6620de5f725cac0013c06301',
|
||||
361000: '50f649f0c8bc0d77ffddd53fd5b896705a8858d3684a3ec876cb320ad26d2c29',
|
||||
362000: 'e058c43c366c9c307ca3ce5a732f0ab2b4e09ff291a067366af25a82fb3cddf1',
|
||||
363000: 'f99d7f0ec93ec7fd74273bb6f8bc4ad8ec0fcda9f6406001668dcf62a088a867',
|
||||
364000: 'e5a69ea8763650542b1f90d6217af00da137a9bb72cd0dbd5379570d1fa696f0',
|
||||
365000: '5d0eb3063474c6bbf9d9581898fcf8c4cd097fe5c6e3875d7cd701a5a0302507',
|
||||
366000: '14da2978f4d4eeff5eb0a6bb663e514095acfe2210039df8dda9b76f522f74ba',
|
||||
367000: '4ad3ebe23fe9b0f407c0765adb69ee973de5b84db29c97167c7a358fe8696bfa',
|
||||
368000: '22db57ccf8c8193ae9fc08abe11f1ea8e869bfd6ab2c0b918820acdaafc19136',
|
||||
369000: '530052d61878d20282a69e0bace41857f7edea6d3e9e92fd0a7781c8b60d208e',
|
||||
370000: 'cd057d0c889c78e06d0eb71287c46deea4340b8c7a4c470dd69020e1a9029c43',
|
||||
371000: '41f32ab47863f0c44b8755ecf9b201aaa92999051ccbf8a5acdee0e12dee4110',
|
||||
372000: '37de29bd97d75a41dc3eb52bed9c09baf069c23120a393888e15e61ac7f81ea2',
|
||||
373000: 'd6b26bb791fdf6d79f59d5c37ef2af0a35149737029157724224d614477527f4',
|
||||
374000: 'de18c6a921838385950b10ed9429579e95a0bdc574f448f457ee486a59a04260',
|
||||
375000: '1f86edf4ae47f409f8de6a16c17c2da063937478a0ccc800633e4d33b531709f',
|
||||
376000: 'a0511050ef086052fe5671d3333c255507b03edbd365cd7ce7cecddb4000f391',
|
||||
377000: '7b8f0060f51fd170296b9e000bb2970097f77ebf0b5872a9ac01be5aa03d7ffa',
|
||||
378000: 'fc5669d194d822c55e141ddc9aa229059e4663169456dc927d43b44a2c8d3c0d',
|
||||
379000: '1c0bf45f052e0123d1a6806fc4a93ac8261ed26772f324c1b3e7c3fa44d7f442',
|
||||
380000: '73b69dcfa0a916a4c495a14d4d43e270b123ba37d267e3eb0d65890336fd3e32',
|
||||
381000: 'e89e3d548193df667c637f774c76f508ec79b7bf1f2ac5d9adf7468006e34454',
|
||||
382000: '6b835973c2efcd564fbd58d96226cb984d5c32dbf63d5b6044a0a11f77c2ccc7',
|
||||
383000: '50b7011656c5903941303c7a19e4ea0fc6bc10456ba85638b0919658556e47cb',
|
||||
384000: '74c509b238c1418ae527d370e8fe922f7639c88ef8d78fc3a45894deeb2252a2',
|
||||
385000: '6f9f6147b06d8fea8c440cdebb3869b7bface7cb2355d66639060056adc9bcc2',
|
||||
386000: '5b59e68fd62751b8be184a5621f1c63f1e9cbd22051487999816df392dca7837',
|
||||
387000: '86d8b9628b83ac63637c86c364001067ee1c3050f395758fd3f0bcab20cef703',
|
||||
388000: '5e570f7b69738285848bd8b7f2fbb82d05e99a72cd4a0faf20a9e3491d2d041c',
|
||||
389000: '78d31bda70842388c1cfb2b83e9e95ded37a0304aaed3292e2687a96c07be9b4',
|
||||
390000: '316da5e2ce79bbc4f4500ea6417271f0ae268f85f86f81223d0af38d1e0e0a67',
|
||||
391000: '45cebf8a25c25aaaac06e915fb3fcc99a84be61a266c2bd8997bd1fe00a7321c',
|
||||
392000: 'c5ff6e24f524bfb1f629245345052c0a0df33ab881ee9d11dcaf64915b2acad3',
|
||||
393000: 'a54b5d4824bae06f91566b73c99a07b812095d7c133d1042ded80211c0402b3f',
|
||||
394000: '0ae76082f8347ddf838ce0ad486f5decf6223e8b8cfc16dfe178b15830807c60',
|
||||
395000: 'dc436c97ee6ed427687e6d48e609d271b125495afdebbcf611c0fd5102ec900e',
|
||||
396000: 'a87871c13ca3119f309a4ed98e4224d4eff01387ea67e86148007ac0d759f2b9',
|
||||
397000: '58c143b302132ad4b38718ebe1c5fa81c1177a30045f1e4a52dfa749e84dde1d',
|
||||
398000: '33089400401a190c9375abb63c6ef489123f7369e5a35766b59e69edaf4e3b93',
|
||||
399000: '656305fae087b86758bc7a01e13051b1aaf4a5c53d9b71e56614217a82b20eea',
|
||||
400000: '7ea274642eac11d02aef589428b181e1c8aed913e248b357917604b00b23c752',
|
||||
401000: '1969a9f64e316e0b4fc864fd7162379821510a944652b8dea747e425c6222807',
|
||||
402000: '3700e1703e7803a4577de7e364a34c3be94d3f933e05c5dfbf6ff68349b6fa31',
|
||||
403000: '8b622599bf4b2d82bc11c897f35536036e6798ac101a6256e14477624147efe2',
|
||||
404000: '5f6f6dcea393ddccdec69b30881ab08e01e1a7e6a50d681883c759b742c12d6e',
|
||||
405000: 'c3fb0507a57da69042e246dc8ecac101f33cf2b1fb53fe2f507ce28a0d82e4b5',
|
||||
406000: '4f034663bb7b84439fa256f7b4ee2a5cf93e0048c1eb476aa2152453ee7be544',
|
||||
407000: 'ff99ff38df6f928242571fed5b03fe0c3cb481b510d3c943e47e26f26440cbda',
|
||||
408000: '89865af5e99e64cef61dd931061bbca7fe355218668f487f06474e46ebf892d8',
|
||||
409000: '1b1d050c0b98d55d9ecb0d1a5b03e75caa352b4e0ce6a57136551000e150c480',
|
||||
410000: 'fe52790595db35c7685e5261de78d139c830af0a5ae73a7e33da21be9ffcc9e6',
|
||||
411000: '1afe55e66e6e16abff282c9d6bcfbc6a3a6e40db3c60323cd44e3b3a5d4ba924',
|
||||
412000: 'b59a4057d73e8037f3e0e9b92310def7900c19917892ebe7689d5f8d77912388',
|
||||
413000: '603f3ca16a6a795d5aa807eac5d6ee7f80a1e2596cd91cf41cf90ee309b2871a',
|
||||
414000: '10e255807a43bf5b95f9af648f1724de1e953a5540819822be5de184ead24ae0',
|
||||
415000: '9faf6b1c33006e3da3ae2c7599c7fdeb5078a8427edd6666290e32aea67f5a23',
|
||||
416000: '973b77e4d491ce0c163ca64f5419b05822eebc2f5d85810f51370f56e0d812b6',
|
||||
417000: 'bf99710d77fe84ae3ef5a371887cacc2ca1fbc5549b04ef9f61cb9d911b06a9f',
|
||||
418000: 'f032acfc2d629fecfa180067b4531f9d72924c5f204cfe9053c6556a334373e8',
|
||||
419000: 'bc9f085612af6a21411fcf51259387d97e6ae1169e4c6f92766a168302a98bac',
|
||||
420000: '9136bd1fa7e8850c52e3cda39a96f754cdc605a69ea49b7d6d959d0cce81efba',
|
||||
421000: 'ec2a78bad1b4b7325d574122ad8c3072d27fe6854a6965e6917a28e0c868a229',
|
||||
422000: '536f4cc5d6ab082d32ccdba1caccb5d4f3145b60ee8538b1e18c6231dcfa23df',
|
||||
423000: 'ac7ad1f97da0e55661bb25d53c27ea72f705d54050f020aad9fd5856a27117cd',
|
||||
424000: 'f840eed1c0fdfba43965bfc175e4d92a5ddd0fd6022dfef988d70a5e7f363116',
|
||||
425000: 'c1a8f9b79a34b76c9ae58e005501dbb1893cfbda8bb25b0d54854657b3ff1a54',
|
||||
426000: 'e8895d8f797026a2f19ade2b548bd2cb655e2d756dcc0a50850fd37148d24ae5',
|
||||
427000: '0ca56cf4fd882583854fca00e339534346d53178c6679f8eb1e7a5dbb7b9e7b0',
|
||||
428000: 'd3ce901843f952df4b63e755a0f58924ae11324aad47afece9a93502dcae7c13',
|
||||
429000: '774cf2d1ba62f20f5e1ce50692f65f0ce35c7456d3988cfbb2e1b753e0ab0f51',
|
||||
430000: 'a07fc310f6201c30f1fa70a082348942c65743d51e3c19492596f6aab4bc9bb5',
|
||||
431000: '7bc7f405bdfc91dd01877ccfd6142c973d81dab12fa188f468abe6c9a30fe0d8',
|
||||
432000: '7fe40e145078c1952f87547156e2147144449971f108164d27bcfb93f1c840d7',
|
||||
433000: 'dbe52d2772d6c0461e53357b71a79096140039ef643876ec2d27e1bdfee7e81f',
|
||||
434000: '94b379e842d4979bbec171bb08881227ac4e9477186ab63f7eee9b4261b92365',
|
||||
435000: 'bc9cd265de563cfe80090ce12085539bc67d406c23e4d41751f191d6b6c63168',
|
||||
436000: '3ff03098d2c20b5cc080363a84d8295510db937272801454d429c4d17a1dbe10',
|
||||
437000: '5b8ebd40170e217eb66990af3574b80f4e66bedd092e44836aa5101ded50c13d',
|
||||
438000: 'f725813181d50c0b20552635ecc578685ab0218619d53ed23493c46fb060ecec',
|
||||
439000: 'cc7cb296ae79850ba2f6f7532c445189452831957f62cb823bc83aa6b51daee9',
|
||||
440000: '757b816392227c9d1a14f7350cab6e77a46e465e7853b1291abdaebcee2bba94',
|
||||
441000: '834515861eb72aa472843a6924fa9d6350630ed56e35181d9fe66e3a46b6952a',
|
||||
442000: 'cac31ac937ebb6f0cf4f3c33f19d66a05bdc11105a98ef12c32b494e19e772a0',
|
||||
443000: '552b03680b5495c7bd6ec58db0b6767e6bde5c5961a02b7b2130ea6116bddbad',
|
||||
444000: '2bea6ef419d9fe81de8bf5cfe21b2603f202e2b9a959d2cfbdcbccbb7a7406c8',
|
||||
445000: 'a4106f2115b356eca92e5e3f45b8ae0e13b82af94bae391942200d1c59d34d17',
|
||||
446000: 'ba49924afd8a40260e546e5e4633e945877ac9d5754c277b77bb154c0c9d0e40',
|
||||
447000: '83ea56135c2b052df0fb09c68cb7a085d876854f34338abaf3cf51760c83c3eb',
|
||||
448000: '806d77789673fa09d24c188ee024e71425df7baf623fda073076f3a9b87ddbce',
|
||||
449000: 'c41d60b93fca1d2adbbfcfa99475a8eda152630d815eb1f6c58cbd15a0c2eb65',
|
||||
450000: '732d315feeb485fcc5055916df17827d80e7f831c43df81ea79123882c4fb356',
|
||||
451000: '202156d29e57c51941cdb7256918128589f5b8dbae5762bbbeacff6eefbbfc71',
|
||||
452000: '0194e8482252f2492e1e2696fa5761f50875d0188310a0cf1d967edffb1b79f5',
|
||||
453000: '9e63570821d8b5cd7de0c5f9fda67300f531d87cd8f778fdc6bee9519b77e01e',
|
||||
454000: '632353fe048e83ebef942d64d550721d29b3fc8083a247b2443f2aa2b706c6fb',
|
||||
455000: '65863feebdbd60d16330000cbae496667a67dd5fa8b8e3b6dd3732843f6222a5',
|
||||
456000: 'bbc554cb3c0281d6853ff2f6c25b1251eb108c09e5a6c4a69a56e8f79c9206e4',
|
||||
457000: 'fe6fdabb84ad1ffa807fd55db29dc95dafeb0e9a5eeb5ce75be8a4d16f9f116c',
|
||||
458000: '949bd40d861e4b297352722b6f01efe9574711fe64b10c2e8dd769ad99cf9655',
|
||||
459000: '9c139c9ccb41fbb3ecee878cf47bb37e28c76816c257f99dd7b23ec1f3eb3ab8',
|
||||
460000: '61c3126d504fb38ea21948d2837368441389155e9675c7f1629eefcd03198f8a',
|
||||
461000: '28ff145fe2abc7cacfb1613d40b45e13580c643ad68c44db81d15125324f4f33',
|
||||
462000: 'e6b532924d07f1ff3976a53397327ccb232d3b2ac6ab090874cfa6ae47b8b315',
|
||||
463000: '15ca651db9b159978ba6b2adfae1669e3042f4311ed683c72838bf337d71274d',
|
||||
464000: '6903539d4f38f728e075441efa96b8013295ac166db50ddd4103fb2bc9937f0f',
|
||||
465000: '6b6c2640411d900c13f5b53a9a993d10fbbd7f03f6fd477e120ba06a7cd67778',
|
||||
466000: 'a05d282fcb9c091a0a96c2ddb3728c1e98eb767fd5311bcc08c9c10b5a0eb24f',
|
||||
467000: 'afb81fdfd562f2cd2d53434e6458c38462f85b65958ecade001bcf2b49f9b28a',
|
||||
468000: '97374586191b3a30654483722f7c6486ef9d9f71aaca44b72146c11d4f4b80c7',
|
||||
469000: '90a829ff58be9ad30024ba1851515cd96d567be1d99154cb966067316d1902af',
|
||||
470000: 'e9fbf2d2291e1b5af5dded3c223611d9f2e31fa6e1d8c075741aa155538bbddc',
|
||||
471000: '3b4513a6a6c05ae9d2ab880a2d1aed3b189e0fca9cfa2bbc7c31ebcbffaed9a4',
|
||||
472000: '32a57820226f672e695823620fccd42a6fe624dd2d88f7e7c76e0e9128883b27',
|
||||
473000: '15cb7e920799939eb01d06b1b713cba3ed31ef92ca037ee98eb8101b6331e5a6',
|
||||
474000: 'cc23e76e68dd726f31e592a4ea5d6aa3cacd827d068317a0a476f16ce468d9f9',
|
||||
475000: '9e7e33a8ee53a4bd94a26b35bb43054b42a238a85a859ed9e4a33db978a128ff',
|
||||
476000: '23c85e4d8edc7939fe0e5a47fa250f0921b366711c34910c394af00a4efd90c6',
|
||||
477000: 'fd9d489fba4175c1fb3b8a0f9bb947287e64da4bee9f8a7fe73411632c820368',
|
||||
478000: 'e287b69696013e70b276843e83af6cb5cd076000cb280e43de29cad7f706dec9',
|
||||
479000: 'e86c7a06d96887e404c052b0fcedf10d40423b02d6b9fa7ccb9827551e5d089f',
|
||||
480000: '74e7ad7d78dc585a5cd1054c0288aefcec0e031cd40f0ebaa0fefc7a325f212a',
|
||||
481000: 'a01f98758cd3e561d02bef30c054ef70405ebdce9e670a7d6165efe3a9030886',
|
||||
482000: 'cc9e22f04e4e2e4f52b8382281092ba50ec99722c9172a0e1c9ad654d26bd4d9',
|
||||
483000: 'c92e278d54111ab2f3906eafab87fdb932307e60836df209856a97cb9e1ae3a8',
|
||||
484000: '18fc89d07631aa01891aa44231dae6cfa43fd4733c193468b033532a4574eb80',
|
||||
485000: 'ab546bb694363e07469dec48b559479d34d724a43865fa8f3a178ddc7b83e8a2',
|
||||
486000: '59926fddc89abcfb55775a58e5e6e453e388f530f720dc61001f1ac59b59cd43',
|
||||
487000: 'b41bcbd91984a3dfe72c87bb7c2ef32ba3eb3cf4318f5d37654459143dceca1b',
|
||||
488000: '94e859b8f01ccbe9fc5bea5c274f252e23599cd444d5eb0962ece97110b86eda',
|
||||
489000: 'fd99b951714162e55d8944e8c3ce9b098c4b74ea24017e4b204f95359202df08',
|
||||
490000: '80b84d18f4bc6d624d4190db94920976926bbe48f2e3e0f4c08106b9f99ac5df',
|
||||
491000: 'b0c5b14b01225b1ed8272dbe9b2def41756cda0a3fe6a1a8e75b0795b72e6a7b',
|
||||
492000: '572dfe57507384278def462488416248a3039c1a76a8c4aa23d1ec706de1e3f6',
|
||||
493000: '75f22a8d4d202914c0f3bffaf95e47eeb82eb00dcd2913af5dd67acf6f8ed8f0',
|
||||
494000: '626e26524f9427582d61f340ee9181d8f238ad8acf96c1862724fab62ede8c8b',
|
||||
495000: '69a11346ba65c79b5fd566768a8b2880a5a29e2552ee8e343bc88bcf43e9ca54',
|
||||
496000: '314c88db22753d317c7b65f872d54d3ff672d04b3781293d59a1ea902811336e',
|
||||
497000: '7bfcf2614b54ac3808894694e8b15940d6740760a2252ba4b47b2926bdd4bf51',
|
||||
498000: 'cc02f90d2fd4e0f378d2ae3bf1011026ca3d7f6642dab316f5fbc2f57aa840d0',
|
||||
499000: '841182d8519cd7b2d8fd01ddd087de892ecfafaacf9d6aa029ce1ed71ee0d538',
|
||||
500000: 'e2742854018ecbfef71d626ab797b7c6137e84ff1e27a05b5da84b2fc7c43145',
|
||||
501000: '07334852d0b542f897164e4d74e699678c850af4e46111d495701a312be21105',
|
||||
502000: '6150771ac25f4185eb94ef6aa15e7f409b6beb411358ba7d982755655d431492',
|
||||
503000: '156d3165eb11d021c46ae5968aeac20f65eb05e68d841bdc98dde156d42fb500',
|
||||
504000: 'e9d474e59bf527d84f031ba41cf471f49e5bf0472ae1b3f48f036813a0b195e6',
|
||||
505000: '261cda40eed5ec5b2a44edcd995b24887a45a417fec3f68507a4d600ef2abc33',
|
||||
506000: 'ffbeca24a8f2b7cd6d64207911e153a2f0bb70d34bb4f073c33197e3a2276558',
|
||||
507000: '14e2664bed708d57478f6fb5a4228e4c26843bd6533056fb38dbf1e7595028a9',
|
||||
508000: '2750e25c3dde233447266f7b35f78d8bff18bbde4ae094d06e38fbf6cd2c6902',
|
||||
509000: '5fc210a72d142dd413bc3b76d5ac772564a5917aef13890e7cee190c61b059d0',
|
||||
510000: '7184fa5628bfe59481bb3542bb4212b535c3ede3c24425f0bf7b1b453c238c6a',
|
||||
511000: '2b4bec04d73f0e524cf12d04bbf7b3859b489275edd047f41b139ee6220074de',
|
||||
512000: 'ebf81767f4d26c98bce62a5f5d1a289da5bba770e88f346e2fbb1d676c155e09',
|
||||
513000: '8bd8a99c65a258ebec809ca76e2cc91b0b4fec5dfe0970736d2fcd706aa660ff',
|
||||
514000: '2dd7cba17366c483819ea090d3196950ee35a4bd9f28acd94575525c851234d5',
|
||||
515000: '33ccf5f5230b9a79cd89eace91cd1c2858f6d040a11494ffb8d51ee934753d57',
|
||||
516000: '30eafb000d199c8d9d1a55c5b09a382a3b10bcbca837e584db6647699ce6a3c0',
|
||||
517000: 'e7ed622512e2483fb8537579ce2d2519053f9a6c8d19c42d665007632d5f74d3',
|
||||
518000: '5180d11f98a25b8698b969c3c9a4b736522397859e7dc00665658f93c6cb8be4',
|
||||
519000: '17ca81b3a3c942c80384c4d2c176b5274c52569b90eef686e077f53a884130c7',
|
||||
520000: 'fe1f552778cd5250ba6556712217b86fd6f1d574143f8ad6238e1d6b67481f97',
|
||||
521000: 'd7ac10bcb8662ab9940c4212dbf0c4b9cf9750e3c04a905efdffbce63f065de7',
|
||||
522000: 'fa7625300aa890ccda58268eaf2daef14539c195f93311099754f955f367413f',
|
||||
523000: 'a55ef67a0e75bea12003609359443a28b42b9037290872d6b1672433cf5c0d34',
|
||||
524000: '08d1d556393bbddf6eda11df852a792515ab9c1d519bd91c48711a0de8faffcc',
|
||||
525000: '76b8691b16f7dafcf980ec3e7c69c2f3caee32057ac8d61b3d3aef0fd394642c',
|
||||
526000: '6d3ecbd7132a725e330f0a566d8d602a9cd96326a36387c1c1fe52f7f6ad975a',
|
||||
527000: '71d08ce7aec6ff50fa1ddc37f6b1574db6e71004e8e111f04a9b715103f68e01',
|
||||
528000: 'e9b103c11db50a121c9e3a1fb4cbbd4fb124b07a67167c5a461a539dac3f932c',
|
||||
529000: 'd2f784907a019c1eb5830a74799562cd6d21570c4591bb038f379897f3296d75',
|
||||
530000: '996bca5a7dfc4a8c7f9f8dfb38b1881fb97893a950f4c8f57462506782ee8ca7',
|
||||
531000: 'c84dea2e7fb0c9c884f8dce525b8b65bb7deb33ab7fdbcf72a5382e9bac2ade5',
|
||||
532000: '3e200d76a92dd1f6fcc420d44ebe446829fa5e0162177acd75a13c98f1477cd1',
|
||||
533000: '37fa7cd81c658ada13d04dcf0e674017c6b7430bb0cd5a080b1b924396377b52',
|
||||
534000: '9483c344293c376d2bd247f2846a5ff5038b27af48ea5badcde101fcd0113dfd',
|
||||
535000: 'c9d4f9982955c07960da1e4f92803ba135f4584416e0767a3567e4d2a21e62dc',
|
||||
536000: 'bc9abe4e4be06aa8f61b599d64923ba88d36886b638f413b12bf87ae852e4010',
|
||||
537000: 'df8307b0097b4882740c18432ab2d7e64486cb7bb70b503923bd630ea277e9e3',
|
||||
538000: '6e2accc5e65a85e0aad098dbc7021bba3cac37c70f6a0ff3135f86b90954bd2c',
|
||||
539000: 'b7d9f50727cb512dbaa1ba073d227805e7adc70f9aee46b9721a65ca3d34a8fb',
|
||||
540000: '19c695a6b8bd256fddfd00c53e6401829ae59ebdc1e71ac234f82ffb2991ed0f',
|
||||
541000: 'cc07c3740f96ec8929c7c8714892a19815fe64f4bcdc4d1bf12cd0a9e1e82125',
|
||||
542000: '23ffec3127ec3e093ec8221fa760510e63fcab31f9f0fb2b48727649e4308980',
|
||||
543000: '6c73b242bb05af125ce4ea324d921ece54c9510980128605aec23cc465dbeb00',
|
||||
544000: '51f215e5f11ae31853c4caaae1d01aac2dbafd00302b70e7198e213d485d5bfe',
|
||||
545000: '2c36150f62c86d050e4c64d83aa0071ef4af0bf8e1e17623c740ea4ee7f3275e',
|
||||
546000: '322e6d77f529412c4b7a1c7778253900f1976f238f96b771845b56aa080f4d23',
|
||||
547000: '1d3413530079389231bdba0a0a7de7c85a4ca991569b2f26a1e48ad2f34a995f',
|
||||
548000: '3576ec0d3ed469c5498229af531fdda290777fa1ab7cd83c49450f730c5ac0d6',
|
||||
549000: 'd2a5ed3d920f273bf891ac40ece534a56f8774c8809ed7d8482ce4b78bdf016e',
|
||||
550000: 'e1f75701ff829339e03c019eef2da1ee4239f38879173a833f81295200e96d29',
|
||||
551000: 'fe8339fb53a33e3b5c4a39cdd64d869f41b4fc88c73fa018eb5245d2a7a01e94',
|
||||
552000: '8c2180cc834d5c14c9801772caa65e82b44cde33da7aac586c81b2bfe77412dc',
|
||||
553000: 'db9b4ea2cbdb21fd48a69da3bac2dbe8cea2a16d45f9252776633661e28fd1d2',
|
||||
554000: 'd09c32f70dd94d737205922618b789204a39668b59fd3ef405ba6d4ba3bd8aea',
|
||||
555000: '6e8efd0190d866972b469cc827330f84f515d6466d8748fc33520e8e0505bf3e',
|
||||
556000: '8ce7cc8aa6ad30971014b332c64ce87303098e64391a9be03d1c621df851c00c',
|
||||
557000: '9959211d82372edae69cfac47cc5c937fecbb6de9e235a5a3954c653f3b1fda6',
|
||||
558000: '4f1d620babaa28a03d02c2ba8b7d1526ae47d45bac1220a1a794cf4a489536be',
|
||||
559000: '3a0f1617c6056a27b1da33f0e4ca80b5b6648341551c19622f7f4b4f016ce8ce',
|
||||
560000: '54877adf522c9abaec851a358ec6fba9957227e0e306d3a425167685380cf6ff',
|
||||
561000: '55db8f8ad8869221ee891244767a7698d48ec61264729cae19008779dadba422',
|
||||
562000: '4ac9c41b29253ed03c45222412c53ddde7b9da63441d06ea91b448d89dc220bb',
|
||||
563000: 'cce11479fe1fcda6af734a03be8ea044df7e2726df67927f5bc76c33728133a2',
|
||||
564000: 'e2e5c52e028ab1be7f5d182a217e16694ffd370930cdafa9443b627e8da256c5',
|
||||
565000: '81488a98ee93b0cf5704cb1f46383456bf5238fdae7bcde4d0826276b5a48235',
|
||||
566000: 'c9afdc804f7b0df600c51b9d596d6107a0cb71a4f8ed3ea764585ffc5f70644d',
|
||||
567000: 'efd3ac4baf1c6e1fb2f8bac40f1416f189c9eb1135330ae4c25f296f012907fa',
|
||||
568000: '225c3db3b83b78fbf9459c7e1e53ae35b01752608d329eefa99247d8a2b9b83e',
|
||||
569000: 'ac98f3fb9084261c19a63fc8f10baeb5b03aaf9ca432d7f763a3a6706f81eefc',
|
||||
570000: '4ee2eae15c9019466bb50d103947bdf60a5397f8fff56154ef782fd1b8579dd9',
|
||||
571000: 'a89f29bfc76988560ccd59a61f8a323fde7c4020abafd577f88d1b0b4c289260',
|
||||
572000: 'eb8380e096c79806443535f2f3b007378e43d2cf3ff56f3b7e6a1b2639ff3da2',
|
||||
573000: '3182ec1555aae0df8d76dd5f37bebac6936e0e0c5609263263eef2e6e87863ef',
|
||||
574000: 'fc45bbadf22417d49e7df89f02df358b85a4cc07adb4010d3bc0334f397b01d6',
|
||||
575000: '6faadf2b0c3a98952bb047a58b83466a223dd323a6e32d68be5e8d064482fa84',
|
||||
576000: '68465ee1d64b65ca71bbeeb611a8b4657d3ce765857d049536047e2ea0490044',
|
||||
577000: 'a019fabcf35e7d4fe15e4e3a499dea2bedc61e4c9cea2803731995def15a9794',
|
||||
578000: 'cc2fb6111fa1a338eeade31574b77a1012ec5948f88e7e5276f7ba4443df3db0',
|
||||
579000: 'c5e7c605c03e56f221397f55bcd164dff21856f7fa48e745e61f2d05d7de5f54',
|
||||
580000: '110fe7150988c930f9c324e00e108050b8be5ca2eaf200f54477073c72feef2a',
|
||||
581000: '0c3adb09dca9a9924fcfe30f16bc70734ec62fd4a306ce65a38b74c5d817ee40',
|
||||
582000: 'fc47433896c47765651d8cb31c88cbf80d0887bf071eaab9d2963af9f8bdfb56',
|
||||
583000: 'c7c518f8634bc7c5cc536c9b77fe119cd6cdce4fafb25d4c0aba53c5bd970e35',
|
||||
584000: '43b2357dd053608b30488aca98f5d9c151a68730fb8d008f1bd1488bfab0c391',
|
||||
585000: 'cdfdb4ebd1c5fc3689b4f013254c832da7bfb12f3440fd327c6b065afd093dce',
|
||||
586000: '48979b814cb6964b20ad8b322c4936c7b241c24fe6d610181c43d1cd003e5d90',
|
||||
587000: 'c262a418368aab465facb8f1a82a392eb74e3fd56760779033a97a3f930630ff',
|
||||
588000: 'b20dc4e0c17a223408a51b958b6b481ddeadad513e046c86c7418a8e96e35992',
|
||||
589000: '0411c5ce49578218dc8e225c11276ed9a9802aac47d8f5837c4eadb3bd72e1dd',
|
||||
590000: 'be8fe8b058722f4919af2834c076e2bfb602654a0343b57eb568d0ed4833fb39',
|
||||
591000: 'affcbbd2a9fa43a64332412d649c2c0bd4b85e90e5e67576f09ee0c207a66cdc',
|
||||
592000: '14a2d414220d240286ec6eddab0b0540ff4d1794b39f7913c00ab005e2ca2a3d',
|
||||
593000: 'a1e2744404626b5e8e101402f50eb751c282ce692d58493ac6fa2efa43254526',
|
||||
594000: '679da754e8f4cfd017ed5c8111361c0f9624104a87a50d077dd7bb5bb97c7ddb',
|
||||
595000: '0c31eeb25d718f96c68de13704a0904c3a524b5936690ddab8b4b38fbabad5f5',
|
||||
596000: 'f930b6516bb01d3365bd0e28aed56a580b4efbb45b0abed5ba941f7a56b545de',
|
||||
597000: '56287fa7bcb53be966ba75c15b92ddf94a7bb51f4ef51037e56ccca936df4d41',
|
||||
598000: '1a6303e68038921fe23644dd10f5cbad4af3648f7cbe8432860f439c84fb226b',
|
||||
599000: 'add270bb4fcb8dc9c254c9130fa037dcedb8ca922c2e35ed403661a896addf5e',
|
||||
600000: '786ea387bc7e1665655f1f683d56c67baeb9bc482bbe4401acd167793d34a508',
|
||||
601000: 'a4c5f398f28cfcdc6e0b6fd467582ff5abfcb4f003cc59ae7703e9467cdd9ee4',
|
||||
602000: '21b19a5a23344325581e7daf0a398b37cac5b4c335237bb8e6c14e270bdc1567',
|
||||
603000: '000eef5b4300cfd804554e199ca6ffcfe73ec8446e82ad32abbc51f45594a368',
|
||||
604000: 'fbf77437dfba9caa4d11b03a3673f2274dbd3d92e704bc5e8ed3f90d99d2c988',
|
||||
605000: '0bdba251f44baa12e2b33b1ba89eb8a5d31ecaf6193b836e6356845149ea3f33',
|
||||
606000: '2dcadcae1cdc68ab6404606937aa9981400e12ae77474462764b8077d391d2d4',
|
||||
607000: 'eff9dfe7fb3e31536dc12c78603f023f8f1f59b2b44a59359e8bfe50b2e22299',
|
||||
608000: '50fdd1223efaaa261f6fe592d7b9e1e32b5a50f78dfee7bc8af19250f8a82776',
|
||||
609000: '5d88ff14c7c36602251061379179ed87c5f7d60d746d8d10fe678a0303ce6596',
|
||||
610000: '5e56249989c623bd7f818ac1e64c5d1b1f3afec3678559d446e356ee5bb3394a',
|
||||
611000: '5a9b15a38dd35b9ef0a028b12ab69b0993eb4bc5577d50c486fdfd032f243a14',
|
||||
612000: '258213d8b7141418f22e303ec43005bae0508a3597fa99c4d59d656d7d768af2',
|
||||
613000: '5ee8b46ebdec5576550540f7fe01c5a91c3ba7bfab26db605ade4e620430cc21',
|
||||
614000: 'c41a592e2042d201bf028bbfef484d264a391db9dc9ef302708e7a39f723cf5c',
|
||||
615000: '70fa6591c0b7bb63665f040b87750668be47058942c79c863bc00232f536d725',
|
||||
616000: 'f4932c1c8fc84e3f4b97336424e27eeb09a65d7347097e983e1ab698a38dcc0a',
|
||||
617000: 'd82bd9b58adf54a50bfd471c66144f2969ab443648fb7bab8299408afc021544',
|
||||
618000: '72f7dfda9bd67b900106e632cf50204fe27cecdeea51d9b0cfde9ba6da8dcd5c',
|
||||
619000: 'ac434b8d723ba498bee878cf589c7d61b74c7a00ea5b7abe70149ba878522332',
|
||||
620000: '519930116d8a8f3b01ba443bbe4fe5346cebc783c632cdd5f62d305f61b6f446',
|
||||
621000: 'fed977a0174f8cdc3e979e7435ffde2f7dbac2d4e1c63593117cfe81f320bfec',
|
||||
622000: '1a6daef00425e6ad136d7bda36cad99baa762f6d788b92f630ed1af33995211e',
|
||||
623000: 'b9f1e2836bd2acae3a9d21516052007005f3f67ae1fc133c1c4c336abe0dce51',
|
||||
624000: '5d4a19f7e7f99b5c39bc844b0a90ebcbd97d88f817f8f2edafca78ae499c3167',
|
||||
625000: 'c1afa80aebb856746da25503eb786f247ab098a94ddc58fc96ca46caee7019c3',
|
||||
626000: '222c2ec50130a658ece226c420c76a8f7a15640033fd89ccaa29bd7c639cb94b',
|
||||
627000: 'de9830e9cc7537c42133153cccf891dcd422203827b1377df748ec27dbfd04a9',
|
||||
628000: '35d44ad84a3d9d548bbc8c91ca734979759e3a03e37153b650f794a0aed1b742',
|
||||
629000: 'c119f3acbd761598673b48eaf0d223e1729b8c8aaef9a715f1bc3dde561a4568',
|
||||
630000: 'ebfda8a47983ced56d40e95ce3be793305899cb92359ee2f9a83f7f6a59a4fe7',
|
||||
631000: '1479c2e6217b97e98f44bbb315ca99d1ab02aad14f52d60f7e6670f5c6d0fbe4',
|
||||
632000: '6ed7f9c90ea95825b1aae91644a68e36b6452e61f05a2821e94399823ad669e3',
|
||||
633000: 'e00eb0bffcf784f9f8fef21b92b9cd98e44db5d790b6eab309d0b11adf910820',
|
||||
634000: '3c71674dab2c2644d910acff4336fb02c2dd8281e30e5c97b9378d50f3d6eac0',
|
||||
635000: 'ee3fbedd791f31de2f820cf37cf5a7bcf1fddff354c1ab5736384f41846894f5',
|
||||
636000: '551b7598f5576c577b48ac7530e56422a5fd31a3860b37994f8489308830e9f6',
|
||||
637000: '531b1a767734f1750077ec9d1ab57fd4430e0154deca2fdb9dc5cfcbe980726c',
|
||||
638000: '3ae3a009b3822211b3bbe9745f006eaf3aeae365e81890205e0223925461af81',
|
||||
639000: '2aa1de8c824fcc1a8f34269010405a0a0fb921fa0f8b998010e4076fd722948c',
|
||||
640000: 'badc624e2ffabc438b7b2c9894e2134e36c2e80af700cebdf0980169c1cfe49a',
|
||||
641000: '0b550e8f9b66ac3a3d0e5a911819643600dec121959fe2b5128cc6c68503e655',
|
||||
642000: 'bfccbe9a0a127d4cf606adb5eb42f3f0ee2ebc780b719d0ec612e590aa7ec47f',
|
||||
643000: '4d44b13390410caf86d145103471469d0b10450166f5b96b8886a440aab5b1cf',
|
||||
644000: '2670950e311d8230e8d3447e97652cb5592248b59968bedb6a98a98c12b8d81d',
|
||||
645000: '6703a25f54614ff79c954438ab9fae5baff32c81fa47306b337f0f87bb11f949',
|
||||
646000: 'c93876f02c788041b400aa42149c176e624eef01bc66b49adf8fcbe255dd1833',
|
||||
647000: '276521b16f6a98f5399a9754621fbd7999b6b94e3cff17cd88d4a1176f431353',
|
||||
648000: '65b6238c589baec511c240cc87cb6ce758ad6cf119f465838a976a853c7f5be6',
|
||||
649000: 'c94f6f7fa7a8afdbecc7c291eab2a3cf218ff1af5117ed1f3f0e60214ec7041d',
|
||||
650000: 'b823008cdb0176fa0522c26b35962b4325fd526217f7eb19464cffcb3c24b4d9',
|
||||
651000: '77024b0bfd3cc7c8d52eff6db4dcec041f8675e6f3c413b7d581bedb1b53bb11',
|
||||
652000: '22a677eef4cfcee12756492eab7e184bda054481d4fc19cc46bc7147c097d118',
|
||||
653000: '66779d92b9fc28fddc46ed14b79b26b3ad7672f0dbc7cf4462d5a49769a89470',
|
||||
654000: '097eb77aede84c35822b3700673efd0129de247cb8ba5ef6b8e6dd7e85bb908b',
|
||||
655000: '83b5b4d4aeef3782c82b9a2fd53937e4523ea15867780e4264688e860fd99593',
|
||||
656000: '99646218016eda7976b45fbb6e891fc4bc130bfb3b5e6dcac8b11e0a4b2f59bd',
|
||||
657000: '9ad1995af3f7f6839a11bf5fcd856f471b200911e1e5a647390a7aa26abc6825',
|
||||
658000: 'fd5a4462737448dcfa9d81802d16f4621ea0b177f538144d836829a536cdd451',
|
||||
659000: '7f6baf86cafbdc820437d63260fcc434694dcf0ab8000307f134fc9c50f437b6',
|
||||
660000: '54a5f2c5a1f1534f23f54c5ac6e3f794ebb62f298dc9ed1aba4112e10cc778cb',
|
||||
661000: '91b46c2d247a43ea916adad7edcc37e18d149fe3ab970eaf39cd060a9c856ce2',
|
||||
662000: 'b49e0df59bfc9dcd32cd1a4b903a12accda70054fbd5f4bc480de0367f254c21',
|
||||
663000: '29d2d3e06f744ec2225659199bd43e446308fe1e1ff16c26e0e89caa468450f1',
|
||||
664000: '50b0665a8c7a9fd1ffb165fbb0148115f83fb7a5f6e9b8ad16c9dff9175d11c8',
|
||||
665000: '35754b82653110c6cb0f9a0e7011ed1ff8b360301e4280e6aa2f6faa5950a369',
|
||||
666000: 'fbc64efa658b00bbb70f5db8265e99465a07627d663c9a716ec8b42bed82338e',
|
||||
667000: 'a046d6957f24ec9f66c73612d3eb483c65e4216fc2761260d9818486cbebe56e',
|
||||
668000: '5d78ef63ea8ce0f66032368effcd56fa443b0f5c488ef8c243d3622113087fbe',
|
||||
669000: '0b1b551500e2c8617150a18deec1e3b9594dd98fc9b97e20dacf2f059f966692',
|
||||
670000: '5fa21e8edbdbb2ccccc2e3aca8e15b0965cd307ea54b6621760e654b53488d33',
|
||||
671000: '90affcb451a786aa42ff93dd1f39961a15cc8c332613f8dc591d9f9c34d359e6',
|
||||
672000: '1a8996b27fb79d43627caf0166f5c8ec72e1e0d09e6c0e17a0d9418d37719afc',
|
||||
673000: '491ec5299fa141a82b438ef5d2429eb5560a0f04412878165c86f9fbb94198af',
|
||||
674000: 'dd680d4c1101a98b6465ac85a63ef12f3ebb1eb88277d37aa341a5b387e2f1b6',
|
||||
675000: 'af8acc98b0fd9ff347acb01801f246c63c6698aff0a0cd69b0f91e17da3742df',
|
||||
676000: 'af24a5759888314e5ef719ee23b9f948b3962ac0b949231a540d45eedaa614f8',
|
||||
677000: '7ec568d4e5ceff99c0e9cefa6407784aa0246845420812bcfd8a5d9c03cb01a8',
|
||||
678000: '845df5d7e1d6f22242ffc4c976937da67ce0880a94c5bf189195d5728f208976',
|
||||
679000: '41f895ce11bd91e09ea8b16102c8f2192cdd2c754e2cb04d33edea688fa7c3b8',
|
||||
680000: 'cbf1e3a313abbe05d90ff1de6d861cf58da50b7f8f7ba43a593a3b6192d65f8a',
|
||||
681000: '1ac184be2b5edfdcc0a73a6a93d3f59a5197bd5bba2ddb1ce737251526998bc9',
|
||||
682000: '0e793523ba0a679dce1e9cf1d1189c5311e27ae8d88c35bce36ddb3c16dc34c4',
|
||||
683000: '08e07172a6b6c3d70c4379b206ed796ea8d916d314d0b6f02539becbb90077d5',
|
||||
684000: '0b67663fc37940b4d40ba88b8b7610776e35f070e4995c5ab09ccd4b86dc1143',
|
||||
685000: '8ae92af68bcc012326889b5ef89b899aa38f65dfd3d9f7dd5b29dba0c5fbebbd',
|
||||
686000: '5223ea0cffd3c59fa6597208425ab55b3a76366fbd27d22508cd208ebf2a2eec',
|
||||
687000: 'e160e55af7fd110905364f980543fca62a123c84c81e9fd38a0aeeebc30a501a',
|
||||
688000: 'fcf3b5e0afae8d665cd6f63dacd2a861aecac20f8b73a682f79cad9523c5c4f3',
|
||||
689000: '772ffab484f07ad91ed45cf6f569eff7440f4243e16e07698e4b7bd4a109e6e3',
|
||||
690000: '5b2a4fc69617ccf1787ee40a0f6d7e0af783bcd856c2c8e0ab747a91a7c68d19',
|
||||
691000: 'e4ba352a759e2425d508a4d5b58595e6ff5eb912d8bfece5a0bc646f61e77084',
|
||||
692000: 'fca2c5b6a721db85278ab55b5e2e39a445e5b1ee5ba69a17044623a6b945d0b2',
|
||||
693000: '99f4365eee70f86499ec26c373922d389cbc5e2a198e96af5d5823ad241748a1',
|
||||
694000: '0e19e76398e65cb01517dcd4ee702c9c04c0fd53cf9468ee539094aa6248e1c8',
|
||||
695000: '5addd92022007d81ced43595458e2eee2903227063af8e9edd75cccbe559930d',
|
||||
696000: 'ba25dfe3467cc9999eb8593d955032475b777cefc8006851f692266ebb83d140',
|
||||
697000: 'e10c1c734038198a99bf970ae89e3a56b2058612e35eac8141d48e147d2d47e0',
|
||||
698000: '472c4de8e57737ca48c3bdbf3c35c37f24a179f5dfa48af89efda3c3d33c131f',
|
||||
699000: 'b359f2e61a3ca3cf8b613045617e38b06767ccff72129f13971faf26a4d08234',
|
||||
700000: 'd0555d53358978ae0abcb09f911c1b3e7b5a282b4ff6d455ca9ad04299666dc5',
|
||||
701000: '5c763b8b4329809553fb58ab279ebbc6639695c45760f0c181168d41da95e6d7',
|
||||
702000: '8a6abdca484fbdac5fd6fb279a435377d964d43b62393b0d5b0cd3baaab1d2c3',
|
||||
703000: '50cf159af75b28c7a95ca990e480ebcd534c4601b03d7c0a979f26f4e33cecd7',
|
||||
704000: '0daf7f9c7eca5c6d91de9ebadb3ef1287c290700bab0aa9f6a2ac4bf42a8a98c',
|
||||
705000: 'a359e7584112eadd7fad5d961f1dee180481852d7706701c2211a98a009c128d',
|
||||
706000: '139ab9432b0c7f62739f818b114d57a59cbf20584230f1198c35f1da62c2ed62',
|
||||
707000: '43e1f4c83d26d419a35adc4dc5bd85c6253ec0faf7e70196bc5d5fac18c51746',
|
||||
708000: '1a05dba8d5d2ef4984c79cc939148ad71535043640fd5644141cdbc512623514',
|
||||
709000: 'fe7a8bd03f742de5227e5d0c8dcfd0b5cc13582703b74b1898c506fc6ffdab04',
|
||||
710000: '26217b4ecadd39e9a14ffb6cbcaab6ee7bf0954efcbbbec4de9bddb461819084',
|
||||
711000: '94aee5cc00e862e4952e573908449f4a48b630285f2f2fb20765d01dfcb68ac6',
|
||||
712000: '21fb256c0f5133634b94f6522cb8d2b0a9b982f1672ba561033be689410e7860',
|
||||
713000: '8d7d7ee4cb0598fecd85c8004b59074bedc3219f2376966e01a7faa92377cdcf',
|
||||
714000: '56a033531fc8dbb7ffe07d7365cce20281e522c3ff9fe7a19d0188af351a5799',
|
||||
715000: '4a6671846cccdc26d6fd1d77db50772315f932c24dc706004e11de68ad1ac387',
|
||||
716000: '4222447b25305bc063b668c0e010b16f9c0802fff9371e9d096ba2174926f073',
|
||||
717000: 'c231fcf5238a34cd15ac735d27b3dd0ee025714e48c408a798f0ba74be0aef77',
|
||||
718000: '0f96aaa30d20fc572d9db26871143a657b706c35a9668d31e3b64280a049787a',
|
||||
719000: '8557c088c4fc4745674c795d4a58aeb3df0dc91bb5ff93cc6268c4a88c64db5a',
|
||||
720000: 'd699e06ee7835c9d687853187125f27145ec91daf394fdee37218b8c34fee9f4',
|
||||
721000: 'c675e891a36425627e20de36f1fbdd5baba7114661f1c45f81f66a3fc55da902',
|
||||
722000: '0c6fedfb6d6c1a77254904fdd2400dedce7d45bdc2271beb77e2087a0ba30d1a',
|
||||
723000: 'a442c320886beccb3d7ea13276dbef8e98e1a47686cba2cdbeb6a2d2883af928',
|
||||
724000: '1592e2d6ac3be7535b44db6cc99080d00b19c6663f52eef3c28eae3dac27ba49',
|
||||
725000: '45b2a800f17b8571172a2658577dbe95b91ae88e611a91ac0c92609b3600f693',
|
||||
726000: 'ed6257a6567665747aa354e93ab7d3e6539d6dd41fced8a2f62cf848e2b30ce0',
|
||||
727000: '4b1a577c6c2358b0344bb1befd9b8e5572b787ec2bdc0bdcd4a150f26b2e2ab7',
|
||||
728000: '448765fbdf6261c376120ff9401db8a8841fcbed466365f982d3bc53775b93ca',
|
||||
729000: '99c2acea0af193d2e10498acd1c6d162d2a804a69157af46817b5ece5ea86491',
|
||||
730000: '94cec967e44f850f512d4240cb8a52ffaf953d0364b0a1dd7604b4a01406e669',
|
||||
731000: '6e63f5019439bc7e27a17a189baad0da8f5724883af3ca35efa0d4e5aaa75b97',
|
||||
732000: '53e1b373805f3236c7725415e872d5635b8679894c4fb630c62b6b75b4ec9d9c',
|
||||
733000: '43e9ab6cf54fde5dcdc4c473af26b256435f4af4254d96fa728f2af9b078d630',
|
||||
734000: 'a3ef7f9257d591c7dcc0f82346cb162a768ee5fe1228353ec485e69be1bf585f',
|
||||
735000: '9bc81abb6c9294463d7fa12b9ceea4f929a5491cf4b6ff8e47e0a95b02c6d355',
|
||||
736000: 'a3b391ecba546ebbbe6e05c5222beca269e5dce6e508028ea41725fef138b687',
|
||||
737000: '0f2e4e43c76b3bf6fc6db9b87adb9a17a05e85110dcb923442746a00446e513a',
|
||||
738000: 'aebdf15b23eb7a37600f67d45bf6586b1d5bff3d5f3459adc2f6211ab3dd0bcb',
|
||||
739000: '3f5a894ac42f95f7d54ce25c42ea0baf1a05b2da0e9406978de0dc53484d8b04',
|
||||
740000: '55debc22f995d844eafa0a90296c9f4f433e2b7f38456fff45dd3c66cef04e37',
|
||||
741000: '927b47fc909b4b55c067bbd75d8638af1400fac076cb642e9500a747d849e458',
|
||||
742000: '97fa3d83eb94114496e418c118f549ebfb8f6d123d0b40a12ecb093239557646',
|
||||
743000: '482b66d8d5084703079c28e3ae69e5dee735f762d6fcf9743e75f04e139fd181',
|
||||
744000: 'f406890d5c70808a58fb14429bad812a3185bdb9dace1aa57de76663f92b5013',
|
||||
745000: '2bd0802cbb8aa4441a159104d39515a4ff6fc8dfe616bc83e88197847c78bcff',
|
||||
746000: '24d090a7b6359db3d5d714a69ddc9a6f2e8ff8f044b723220a8ba32df785fd54',
|
||||
747000: '07c4ce9ce5310ee472cf753ddb03c39c5fee6c910d491daffd38615205411633',
|
||||
748000: 'ea913798c0f09d0a27eae7c852954c2c88b8c3b7f23f8fba26b68a3952d0ffde',
|
||||
749000: '23f256adebfe35d49ba84ad49f3f71fc67f7745091c91f22e65f1cc2e23b8f2c',
|
||||
750000: '96db12ee3a295f3d5c56d244e6e7493f58c08d3427e379940e5d4f891a41ec26',
|
||||
751000: 'cedaf12415dac1314942e58ced80830b92fbfabc41f42a0b0f054f0672ef9822',
|
||||
752000: '293606bcd9fbbee5584724301b2cf86bb69204820023e1fb46c238ddfbc660ab',
|
||||
753000: 'f4d43cbb38b7d97919dedc0f5a6dc8007896c4f443b76f3e5693e25bc46760cf',
|
||||
754000: 'fcaad22fd815311280fe451086516375d1d9d92b2990c7c351407df5aa19011e',
|
||||
755000: 'b9276f10d1844cb5b0308766c8db960490ac34a73c4653d0a91202789a6ccb9b',
|
||||
756000: '2fe5581f1110c1c8dcea46cad647551bd6bd640cb37738d863e189bd8f368347',
|
||||
757000: 'b9d915f366f0b010429a52245b0fb02774157eb9fd8f66bce32dcd3acc71c2a1',
|
||||
758000: '62d1854fc15db56b5d0e05ceeb54c1297966bf9dc7f7a0a14b42c059fc485d1b',
|
||||
759000: 'f4ca9f69d16d092f4a0ea5102e6343b21204c4ea9cd9b22cddd77dbb5d68ade3',
|
||||
760000: 'df3bb86641330d8cc7f55a2fd0da28251219e95babe960a308b18e08a7d88fc8',
|
||||
761000: 'a93029475de4bc7569b6ae802d658cd91c84cc253772712a279f140a6c3b91b1',
|
||||
762000: '307e289dc6ec8bcd62ca8831e4159d5edd780f2fae55ba55dd446225450f46f8',
|
||||
763000: '293f73514abca24f374473bd0394179812952a04ea13dc60ef5ada5331fa274f',
|
||||
764000: 'dd8b082db9281e3d9bacf15d6b352fda186d2d2923c7731844d0d4764dd71db8',
|
||||
765000: '201239e562d2571bf47347b3522fff89632aecea3b2d8cef05151f88b2b0bcdb',
|
||||
766000: '4a55a538b51b5650979e64521998cd5c5ad055ba9f3ac0e3e2a28febc6cc2798',
|
||||
767000: '3916666f2adbb05ea98ec1961f9546b9afa0f6910ec95e42ce37267f2ae4f79c',
|
||||
768000: 'dc0ad881eedcb5fd4954238f462080d6e7636b058d481698ed1c077e0ce2207e',
|
||||
769000: 'eaf10a1e1ec6e129289b8479a05df03e0808f1f0946f1995de6524e9ebe7a461',
|
||||
770000: '7200c64f22e32de7f999583361c933680fc9a2ffcb9a5ab73d3076fd49ec7537',
|
||||
771000: 'd883111a2eeacff80ce31df35ab6c943805b9e48877b413fccf371e5dbfa7fb2',
|
||||
772000: '3977d3c60edb9c80c97bb2b759b1659cbb650ad2d3a6f61d2caec83f1b2ae84c',
|
||||
773000: '9c7175fb8646a1a82383b4c534fd01bcf92d65c43d87ae854d51a784b04dc77e',
|
||||
774000: 'e0e92485f86e5fffa87b3497424e43b02a37710517d9d3f272392e8cdc56e5e9',
|
||||
775000: '6395229113d3aa2105afbaeb8b59621a536fc61fe272314b2fc3bdda98dd66cc',
|
||||
776000: 'b4b00207328b5f032bd4f0b634f91323ff520ada8c8bfec241b23c8e4bfd5a4e',
|
||||
777000: '14cdc6f5f7b4bd5bad745dfe6fcd114e9194026412a2e1b3f345be2eef433d16',
|
||||
778000: 'd3cd7b68be504c32117b670d38d59d44b02dcf3d65811efc2ca5531d902623cc',
|
||||
779000: 'afcd220e4040cb5f92d4b38fc204e59822df2218f767f2c4b33597b238a35f77',
|
||||
780000: '78252a9cfc289a70192ed8dd3dddeb1b9a4f9b8eff9a5d0ac259b3254472cf68',
|
||||
781000: '02ebc3f17d947481a311b4771c254f1e002b6a9198d4a5258ce6c13165aadddc',
|
||||
782000: '8dd9f1f372ee6d688a0bcdc3b342c77804ba5a646a218be4bc2aa02d846206c0',
|
||||
783000: 'e46b0d02ec2ef488fae455665e107520e1bd2b4f35ca52af7ad8addd2f72fa73',
|
||||
784000: '9ee8a8de94231e3ae3a610b82fdbca48dc14d9b80791d20af6c365a31822df6f',
|
||||
785000: '21e1cc12def8173a50158b2833bd91a62140c61646f5e08aecaee3e6da20735e',
|
||||
786000: 'b3e659f84d73de42888cc0f2b69bae71dd5fa6756a437a4b21958b182faa316e',
|
||||
787000: 'a9be7ba00ea6a9ea6bd03d8412ec014ca7e8cda6bdc33382f165e702811b8836',
|
||||
788000: 'a4c14729f8a68c03f5a0ccd890ac6a92b39c143f1f752fe81ad051eb52d8dce0',
|
||||
789000: '5cf66d224e5645097efc9c3c0392b51c8ca8ea1295151921a7912a2f04ee1274',
|
||||
790000: '676769ade71c33bc102bce416e66eb2c6794b03d7b8f5a590c87c380da463775',
|
||||
791000: '0228e074451797bf6bfbc941bcafcbadc972d32e4e1e0c5da015513f65714217',
|
||||
792000: '0fa3d00a1f19c5ac060e10a410cf7cea18eac5f89018d79ce51ac3fc66bbb365',
|
||||
793000: '5f68d0868b424e32f5ce3d8e7d9f18979da7b831b8ef4e3974d62fb20ff53a97',
|
||||
794000: '34508c56423739c00a837801b654b07decb274d02b383eff396d23c4d64bc0e9',
|
||||
795000: '7f70910c855d1fd88cd7f9be8a3b94314ee408a31a2da6301404bf8deb07c12c',
|
||||
796000: 'b74ab8813b1d2a0967fea0e66597572e5f0b5a285e21f5150fcc9d5f757de130',
|
||||
797000: 'bba27b1491d907ab1baa456cb651dc5b071231b1b6ad27b62d351ca12c25dbfd',
|
||||
798000: 'e75dcb15b2fc91f02e75e600dde9f6f46c09672533bc82a5d6916c4a2cd8613a',
|
||||
799000: 'adf62c826a3e0b33af439a7881918ae4ce19c5fb2ca37d21243415f7d716aa65',
|
||||
800000: 'd8f0ca13a8c8a19c254a3a6ba15150a34711dca96f2d877162cc44aa2acfb268',
|
||||
801000: '2a8c7104c4040a2bc31913ae25e9361df5bac9477368c708f86c1ca640480887',
|
||||
802000: '1f3b09d3561c4a8a056b263289bd492dc6c0d604c3fa195935e735d1c0ddc40e',
|
||||
803000: '037769628c40a701fdb4b16d79084b8fbb319fde79770a7ac842f3cdc813099e',
|
||||
804000: 'a0c6a089e5fa1e3589ca282085fe7201a5705776d81b257ffd252b2947fa6428',
|
||||
805000: 'b2ac99bfc4a488e7b7624b31ee061991a6dd0881bb005cd13f3dd2e66a08fe19',
|
||||
806000: 'ffe63cb999a278280b80a667d2dcb60c40e43a53f733914d8bec808b694ebf83',
|
||||
807000: 'eddb09fc6c4869a59b520d0befb1fb6ac952333f3cc5de086539c85ea8558778',
|
||||
808000: '0f4fb3f9172e52897ea992d9f3a2024126c4d2e63e9888739f11fb1f5e4c1f46',
|
||||
809000: '9641dd720d23ced2f1cb6e5cf46ac4e547afb9f56263c4cf58e3b19d407cf401',
|
||||
810000: 'de6dc953acd7e5ef213b3aaf1c4a9ee1d5b756bfce5525ee105214647e243a85',
|
||||
811000: 'c52c83712ca12b24b2db1b4a575e7f352b1d560cbf702e121a03bdca9e8be23d',
|
||||
812000: '83143734bb965318a53a38a7e403dcdb3e3fadedb01ab12c370417fc2a0655c0',
|
||||
813000: 'e480deff10c5a84fc957e3aed936690e24b74dd08fa8858a8a953c2f7383b914',
|
||||
814000: '810d33afcee07b9abe16c6cdc3a041038daa131c476b0daf48a080007f08b490',
|
||||
815000: 'b4aeb9e16fddd27844b2d56bc2b221134039bb5642c9e9ba88372afbdeac3972',
|
||||
816000: '86e73b67aae3d248011b8f66ed414cb8a9ba4b2a3cf7e32773cfbff055d719b7',
|
||||
817000: '3ebb8b83752b48242016cb682f0f6bd14e15371bf1163a5933193eaa0edeb351',
|
||||
818000: '4d925e17f642f220bbf317d3d5355d2f41fbce325f190f8c3b32dc0b337d24d6',
|
||||
819000: 'b9cc126d620f6b99d90a00d35957b0e428aaaa7c986bc9e816a60e4334572961',
|
||||
820000: '9c2f8c142bed1f94dca29276f7c83958be8cfe11773bb9b56c808fbcf7d3b1f8',
|
||||
821000: 'e5509eb98895cfa12a8da5d54c1df3f52472ffcbdf707adbf84a4a9c5d356203',
|
||||
822000: '764aada4802ebfe4ef935ab50af06a4f83aa556c49fdde3d9e12e1abd230c16b',
|
||||
823000: '1dbd745c2e96a365d865f990d109137d32d42977f503af55d8c00b109d31d3c3',
|
||||
824000: '954304a0b0c8f549c3bffd5ff46b5b8f05b0f0fde2a36f24fd5af9d774fb3079',
|
||||
825000: '17808b14f2056c1a5d46cb7617e9de9be6a1a6084edbc1bdb778586467a72297',
|
||||
826000: '3ca1167d4cac8b187829b23001b438617c43704b42462c4eb001b0d434cb9651',
|
||||
827000: '246d1607245e4a202f420393ac2e30e9cbf5eb5570dc997073b897f6d8643023',
|
||||
828000: '1764730a8dc3e89d02d168ff6bb54e8c903820b74711af6ff27bd0c8545577e7',
|
||||
829000: 'd9f3ab0cd823c6305bd8b95a96188bb4f2ca90b4d66c5d12293e8b6192bac0f2',
|
||||
830000: 'd4ff51f0092b04aedf8d39937680d8e8309b1be21d36e7833ed36f8e30aad6ea',
|
||||
831000: '3e92e76721b962396dce52993fa7606552f0907b38f7b2bd7b21ada98c145f47',
|
||||
832000: 'df12fcdb4cbe53ba627ace6de898298de175f8671d3d90170732d110fcdc34b8',
|
||||
833000: '25167ff38ae4a5964b618cabe0a12d4de62ac7a4c47448cdb4499e09e108d5b9',
|
||||
834000: 'd31f5309ea179a1e386e835fc372e47dcda6871a3a239abfba50c4f368994f13',
|
||||
835000: 'aff7e8dd3e55ea807fcbe284014075f420b3a23f1b0eb47bacdc1c91d2899813',
|
||||
836000: '3b5ac6d64c470739bb17d1544a285affb40f2d33e92687e5ba7c5ac602e0d72a',
|
||||
837000: 'd5619cbfe4f27c55f2bf9351b4891636cf64fef88212a5eeeae7bd3de47fe0bd',
|
||||
838000: '1f9102a49c6ac470cb5d0050e5300b1443840d6d65719b835e3bea484aafb2ec',
|
||||
839000: '3f63e391f0fbc5787fbe4ace3bada3816261294ea1c6ee435001801023682f90',
|
||||
840000: '777894fd12bd0d6dee7bcde2995c68e55e7094e3122da38571e4b6c4304b75e0',
|
||||
841000: 'ceb0c598c788e25e43e25aa4beff5c7377035824844cf1675eaea537074df028',
|
||||
842000: '8661cf2065dc713d2ba043f0b81f0effcc940eeb3e91906a21ff22c210561dcd',
|
||||
843000: '0dc2766f90415009d0c86bedffee6ebcf58042eb08262c0c67c4e9ed86b2aec8',
|
||||
844000: '26d072da864cab268a12794977b04ec44fb69ef3978e2342e82225974dac54dd',
|
||||
845000: '95e93bb60be8d5f07a1f4d26290c914957a82fc9d26ae8a3f20082eda27406ff',
|
||||
846000: 'f1bdc39af7705e58ab8b6c31dc70dce1e115db1cfd8cc9b037949dfbec82a59a',
|
||||
847000: 'f5f10f06396ecf2765d8a081141d489737c1d8d57c281f28f57c4cb2f90db883',
|
||||
848000: '331b8ef08605bae8d749893af9ed54f0df4f07a5a002108a2a0aea82d0360979',
|
||||
849000: '75b5f6233ab9a1bbc3c8b2893e5b22a0aa98e7ea635261255dc3c281f67d2260',
|
||||
850000: '5d7e6fe83e0ea1910a54a00090704737671d6f44df4228e21440ad1fc15e595f',
|
||||
851000: '7822db25d3ff0f6695ee38bad91edf317b5c6611673d28f1d22053110bb558be',
|
||||
852000: '2f0effad83a3561fc1a2806a562786a641d9ddb18d16bb9308006e7d324a21e9',
|
||||
853000: 'f603b2eaff11d5296377d990651317d40a1b2599ad2c5250eab131090f4b9458',
|
||||
854000: '34d59b26a50f18a9f250736d0f2e69d28b7e196fbef9b8a26c6b0b75c16aa194',
|
||||
855000: '76dd1ffff3946c0878969886fcf177ce5ab5560df19ddf006f9bcb02ae3e4e4f',
|
||||
856000: '74ff0b6f64e9dd5802fec2aac1d3ae194d28b9264114adaf0a882b46c8c918fe',
|
||||
857000: '7b5badfa2e4f40aa597a504d7ebe83c3705a2c6169a8c168ce293db223bc2d32',
|
||||
858000: '2bb0767a0f72b20d45ecfc3e34517dbda16d85758e040cf0e147f4cbd0cc57ac',
|
||||
859000: '3d741b9c365a91ed76f85824b94d19ec19b608d232660840ba59c7aa4b2cb67f',
|
||||
860000: 'd481a5a117878c0e3acd1f5844e150fb30e617577947d9846b1d214d703b71b0',
|
||||
861000: '54033424e488a3f1ad6946d4a6d9acb48465d6b1dbe8e1c2504a54cc84d7cad4',
|
||||
862000: '464bc3820a8cc8844dc9e26c388009e9982c656d46ef4b4fd0a2cb0e4eea0aaa',
|
||||
863000: 'd1aa94be2174f66780c4f226b9da3f6712b0f37af8dec33360bea83ca261b342',
|
||||
864000: '8c16008f11de5bc395d88cd802514ff647450f1bc136724b9aaf2ccce10a494f',
|
||||
865000: '3dae86012e97a201e2e1a47c899001ac00f78dc108026ed7c4194858c6c6dd5a',
|
||||
866000: 'afe5b0ccab995e1a1fa25fbc24c1d4b1a92c43042d03395f8743dcd806e72fd8',
|
||||
867000: 'c83716ac171aa9ab0d414833db340fa30e82bfda6cc616d3038529caab9b5600',
|
||||
868000: '8c409fe03cd35ef2d8e366818788b40eaeb4c8f6ae91450d75f4a66ca5f69cad',
|
||||
869000: '1d47909ceba790b8e1ce2e9902ee2775ea99e58efdb95668f9803a8ccf95f286',
|
||||
870000: '9adf5da1476388f053aa42de636da169d1cf1c9652cdf7cd9ad4fb18a0eb3388',
|
||||
871000: '8ad57fb1e74bcba0b5614fbac003be2bb32275dd85b38f2d28a0585005a99cfc',
|
||||
872000: '84a32e92012a356106e9657da8dab1a5491ea588fc29d411c69b20680c666420',
|
||||
873000: 'adf5921bbbfaa43929f67e6a070975313b77b456e262c700a27be611fceb17ae',
|
||||
874000: '09eaa7c4b18c79a46a2895190333f72336826223d5c986849a06f5153f49f2a5',
|
||||
875000: '235d7e4f31966507312149ea4c5e294aa84c695cf840117f0ef5963be7a0bda1',
|
||||
876000: '9aa9cb806ccbec0475ac330b496c5b2edeba38ba3f1e13ddd54a01457634a288',
|
||||
877000: 'c1e7f9b2b20bb1c4c0deadbc786d31fdf36f262325342aa23d1a66e2846b22bc',
|
||||
878000: 'ee0d2b20ac28ce23ab38698a57c6beff14f12b7af9d027c05cc92f652695f46b',
|
||||
879000: '0eb0810f4b81d1845b0a88f05449408df2e45715c9210a656f45278c5fdf7956',
|
||||
880000: 'e7d613027e3b4ca38d09bbef07998b57db237c6d67f1e8ea50024d2e0d9a1a72',
|
||||
881000: '21af4d355d8756b8bf0369b2d79b5c824148ae069026ba5c14f9dd6b7555e1db',
|
||||
}
|
||||
@@ -5,7 +5,7 @@ from lbry.wallet.transaction import OutputEffectiveAmountEstimator
|
||||
|
||||
MAXIMUM_TRIES = 100000
|
||||
|
||||
STRATEGIES = ['sqlite'] # sqlite coin chooser is in database.py
|
||||
STRATEGIES = []
|
||||
|
||||
|
||||
def strategy(method):
|
||||
|
||||
@@ -6,7 +6,6 @@ COIN = 100*CENT
|
||||
TIMEOUT = 30.0
|
||||
|
||||
TXO_TYPES = {
|
||||
"other": 0,
|
||||
"stream": 1,
|
||||
"channel": 2,
|
||||
"support": 3,
|
||||
@@ -15,13 +14,9 @@ TXO_TYPES = {
|
||||
"repost": 6,
|
||||
}
|
||||
|
||||
CLAIM_TYPE_NAMES = [
|
||||
'stream',
|
||||
'channel',
|
||||
'collection',
|
||||
'repost',
|
||||
]
|
||||
|
||||
CLAIM_TYPES = [
|
||||
TXO_TYPES[name] for name in CLAIM_TYPE_NAMES
|
||||
TXO_TYPES['stream'],
|
||||
TXO_TYPES['channel'],
|
||||
TXO_TYPES['collection'],
|
||||
TXO_TYPES['repost'],
|
||||
]
|
||||
+188
-721
File diff suppressed because it is too large.
Load diff
+68
-138
@@ -1,18 +1,16 @@
|
||||
import base64
|
||||
import os
|
||||
import struct
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import zlib
|
||||
from datetime import date
|
||||
|
||||
from io import BytesIO
|
||||
from typing import Optional, Iterator, Tuple, Callable
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Optional, Iterator, Tuple
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from lbry.crypto.hash import sha512, double_sha256, ripemd160
|
||||
from lbry.wallet.util import ArithUint256, date_to_julian_day
|
||||
from .checkpoints import HASHES
|
||||
from lbry.wallet.util import ArithUint256
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -34,50 +32,25 @@ class Headers:
|
||||
max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
|
||||
target_timespan = 150
|
||||
checkpoints = HASHES
|
||||
first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot
|
||||
timestamp_average_offset = 160.6855883050695 # calculated at 733447
|
||||
checkpoint = (600_000, b'100b33ca3d0b86a48f0d6d6f30458a130ecb89d5affefe4afccb134d5a40f4c2')
|
||||
|
||||
validate_difficulty: bool = True
|
||||
|
||||
def __init__(self, path) -> None:
|
||||
self.io = None
|
||||
if path == ':memory:':
|
||||
self.io = BytesIO()
|
||||
self.path = path
|
||||
self._size: Optional[int] = None
|
||||
self.chunk_getter: Optional[Callable] = None
|
||||
self.known_missing_checkpointed_chunks = set()
|
||||
self.check_chunk_lock = asyncio.Lock()
|
||||
|
||||
async def open(self):
|
||||
self.io = BytesIO()
|
||||
if self.path != ':memory:':
|
||||
def _readit():
|
||||
if os.path.exists(self.path):
|
||||
with open(self.path, 'r+b') as header_file:
|
||||
self.io.seek(0)
|
||||
self.io.write(header_file.read())
|
||||
await asyncio.get_event_loop().run_in_executor(None, _readit)
|
||||
bytes_size = self.io.seek(0, os.SEEK_END)
|
||||
self._size = bytes_size // self.header_size
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000
|
||||
if bytes_size % self.header_size:
|
||||
log.warning("Reader file size doesnt match header size. Repairing, might take a while.")
|
||||
await self.repair()
|
||||
else:
|
||||
# try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok)
|
||||
await self.repair(start_height=max_checkpointed_height)
|
||||
await self.ensure_checkpointed_size()
|
||||
await self.get_all_missing_headers()
|
||||
if not os.path.exists(self.path):
|
||||
self.io = open(self.path, 'w+b')
|
||||
else:
|
||||
self.io = open(self.path, 'r+b')
|
||||
|
||||
async def close(self):
|
||||
if self.io is not None:
|
||||
def _close():
|
||||
flags = 'r+b' if os.path.exists(self.path) else 'w+b'
|
||||
with open(self.path, flags) as header_file:
|
||||
header_file.write(self.io.getbuffer())
|
||||
await asyncio.get_event_loop().run_in_executor(None, _close)
|
||||
self.io.close()
|
||||
self.io = None
|
||||
self.io.close()
|
||||
|
||||
@staticmethod
|
||||
def serialize(header):
|
||||
@@ -124,92 +97,23 @@ class Headers:
|
||||
return new_target
|
||||
|
||||
def __len__(self) -> int:
|
||||
if self._size is None:
|
||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||
return self._size
|
||||
|
||||
def __bool__(self):
|
||||
return True
|
||||
|
||||
async def get(self, height) -> dict:
|
||||
def __getitem__(self, height) -> dict:
|
||||
if isinstance(height, slice):
|
||||
raise NotImplementedError("Slicing of header chain has not been implemented yet.")
|
||||
try:
|
||||
return self.deserialize(height, await self.get_raw_header(height))
|
||||
except struct.error:
|
||||
raise IndexError(f"failed to get {height}, at {len(self)}")
|
||||
|
||||
def estimated_timestamp(self, height, try_real_headers=True):
|
||||
if height <= 0:
|
||||
return
|
||||
if try_real_headers and self.has_header(height):
|
||||
offset = height * self.header_size
|
||||
return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0]
|
||||
return int(self.first_block_timestamp + (height * self.timestamp_average_offset))
|
||||
|
||||
def estimated_julian_day(self, height):
|
||||
return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False)))
|
||||
|
||||
async def get_raw_header(self, height) -> bytes:
|
||||
if self.chunk_getter:
|
||||
await self.ensure_chunk_at(height)
|
||||
if not 0 <= height <= self.height:
|
||||
raise IndexError(f"{height} is out of bounds, current height: {self.height}")
|
||||
return self._read(height)
|
||||
return self.deserialize(height, self.get_raw_header(height))
|
||||
|
||||
def _read(self, height, count=1):
|
||||
offset = height * self.header_size
|
||||
return bytes(self.io.getbuffer()[offset: offset + self.header_size * count])
|
||||
|
||||
def chunk_hash(self, start, count):
|
||||
return self.hash_header(self._read(start, count)).decode()
|
||||
|
||||
async def ensure_checkpointed_size(self):
|
||||
max_checkpointed_height = max(self.checkpoints.keys() or [-1])
|
||||
if self.height < max_checkpointed_height:
|
||||
self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000))
|
||||
|
||||
async def ensure_chunk_at(self, height):
|
||||
async with self.check_chunk_lock:
|
||||
if self.has_header(height):
|
||||
log.debug("has header %s", height)
|
||||
return
|
||||
return await self.fetch_chunk(height)
|
||||
|
||||
async def fetch_chunk(self, height):
|
||||
log.info("on-demand fetching height %s", height)
|
||||
start = (height // 1000) * 1000
|
||||
headers = await self.chunk_getter(start) # pylint: disable=not-callable
|
||||
chunk = (
|
||||
zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000)
|
||||
)
|
||||
chunk_hash = self.hash_header(chunk).decode()
|
||||
if self.checkpoints.get(start) == chunk_hash:
|
||||
self._write(start, chunk)
|
||||
if start in self.known_missing_checkpointed_chunks:
|
||||
self.known_missing_checkpointed_chunks.remove(start)
|
||||
return
|
||||
elif start not in self.checkpoints:
|
||||
return # todo: fixme
|
||||
raise Exception(
|
||||
f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead."
|
||||
)
|
||||
|
||||
def has_header(self, height):
|
||||
normalized_height = (height // 1000) * 1000
|
||||
if normalized_height in self.checkpoints:
|
||||
return normalized_height not in self.known_missing_checkpointed_chunks
|
||||
|
||||
empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d'
|
||||
all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b'
|
||||
return self.chunk_hash(height, 1) not in (empty, all_zeroes)
|
||||
|
||||
async def get_all_missing_headers(self):
|
||||
# Heavy operation done in one optimized shot
|
||||
for chunk_height, expected_hash in reversed(list(self.checkpoints.items())):
|
||||
if chunk_height in self.known_missing_checkpointed_chunks:
|
||||
continue
|
||||
if self.chunk_hash(chunk_height, 1000) != expected_hash:
|
||||
self.known_missing_checkpointed_chunks.add(chunk_height)
|
||||
return self.known_missing_checkpointed_chunks
|
||||
def get_raw_header(self, height) -> bytes:
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
return self.io.read(self.header_size)
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
@@ -219,9 +123,9 @@ class Headers:
|
||||
def bytes_size(self):
|
||||
return len(self) * self.header_size
|
||||
|
||||
async def hash(self, height=None) -> bytes:
|
||||
def hash(self, height=None) -> bytes:
|
||||
return self.hash_header(
|
||||
await self.get_raw_header(height if height is not None else self.height)
|
||||
self.get_raw_header(height if height is not None else self.height)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -230,18 +134,44 @@ class Headers:
|
||||
return b'0' * 64
|
||||
return hexlify(double_sha256(header)[::-1])
|
||||
|
||||
@asynccontextmanager
|
||||
async def checkpointed_connector(self):
|
||||
buf = BytesIO()
|
||||
try:
|
||||
yield buf
|
||||
finally:
|
||||
await asyncio.sleep(0)
|
||||
final_height = len(self) + buf.tell() // self.header_size
|
||||
verifiable_bytes = (self.checkpoint[0] - len(self)) * self.header_size if self.checkpoint else 0
|
||||
if verifiable_bytes > 0 and final_height >= self.checkpoint[0]:
|
||||
buf.seek(0)
|
||||
self.io.seek(0)
|
||||
h = hashlib.sha256()
|
||||
h.update(self.io.read())
|
||||
h.update(buf.read(verifiable_bytes))
|
||||
if h.hexdigest().encode() == self.checkpoint[1]:
|
||||
buf.seek(0)
|
||||
self._write(len(self), buf.read(verifiable_bytes))
|
||||
remaining = buf.read()
|
||||
buf.seek(0)
|
||||
buf.write(remaining)
|
||||
buf.truncate()
|
||||
else:
|
||||
log.warning("Checkpoint mismatch, connecting headers through slow method.")
|
||||
if buf.tell() > 0:
|
||||
await self.connect(len(self), buf.getvalue())
|
||||
|
||||
async def connect(self, start: int, headers: bytes) -> int:
|
||||
added = 0
|
||||
bail = False
|
||||
for height, chunk in self._iterate_chunks(start, headers):
|
||||
try:
|
||||
# validate_chunk() is CPU bound and reads previous chunks from file system
|
||||
await self.validate_chunk(height, chunk)
|
||||
self.validate_chunk(height, chunk)
|
||||
except InvalidHeader as e:
|
||||
bail = True
|
||||
chunk = chunk[:(height-e.height)*self.header_size]
|
||||
if chunk:
|
||||
added += self._write(height, chunk)
|
||||
added += self._write(height, chunk) if chunk else 0
|
||||
if bail:
|
||||
break
|
||||
return added
|
||||
@@ -249,21 +179,20 @@ class Headers:
|
||||
def _write(self, height, verified_chunk):
|
||||
self.io.seek(height * self.header_size, os.SEEK_SET)
|
||||
written = self.io.write(verified_chunk) // self.header_size
|
||||
# self.io.truncate()
|
||||
self.io.truncate()
|
||||
# .seek()/.write()/.truncate() might also .flush() when needed
|
||||
# the goal here is mainly to ensure we're definitely flush()'ing
|
||||
self.io.flush()
|
||||
self._size = max(self._size or 0, self.io.tell() // self.header_size)
|
||||
self._size = self.io.tell() // self.header_size
|
||||
return written
|
||||
|
||||
async def validate_chunk(self, height, chunk):
|
||||
def validate_chunk(self, height, chunk):
|
||||
previous_hash, previous_header, previous_previous_header = None, None, None
|
||||
if height > 0:
|
||||
raw = await self.get_raw_header(height-1)
|
||||
previous_header = self.deserialize(height-1, raw)
|
||||
previous_hash = self.hash_header(raw)
|
||||
previous_header = self[height-1]
|
||||
previous_hash = self.hash(height-1)
|
||||
if height > 1:
|
||||
previous_previous_header = await self.get(height-2)
|
||||
previous_previous_header = self[height-2]
|
||||
chunk_target = self.get_next_chunk_target(height // 2016 - 1)
|
||||
for current_hash, current_header in self._iterate_headers(height, chunk):
|
||||
block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header)
|
||||
@@ -302,30 +231,28 @@ class Headers:
|
||||
height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}"
|
||||
)
|
||||
|
||||
async def repair(self, start_height=0):
|
||||
async def repair(self):
|
||||
previous_header_hash = fail = None
|
||||
batch_size = 36
|
||||
for height in range(start_height, self.height, batch_size):
|
||||
headers = self._read(height, batch_size)
|
||||
for start_height in range(0, self.height, batch_size):
|
||||
self.io.seek(self.header_size * start_height)
|
||||
headers = self.io.read(self.header_size*batch_size)
|
||||
if len(headers) % self.header_size != 0:
|
||||
headers = headers[:(len(headers) // self.header_size) * self.header_size]
|
||||
for header_hash, header in self._iterate_headers(height, headers):
|
||||
for header_hash, header in self._iterate_headers(start_height, headers):
|
||||
height = header['block_height']
|
||||
if previous_header_hash:
|
||||
if height:
|
||||
if header['prev_block_hash'] != previous_header_hash:
|
||||
fail = True
|
||||
elif height == 0:
|
||||
else:
|
||||
if header_hash != self.genesis_hash:
|
||||
fail = True
|
||||
else:
|
||||
# for sanity and clarity, since it is the only way we can end up here
|
||||
assert start_height > 0 and height == start_height
|
||||
if fail:
|
||||
log.warning("Header file corrupted at height %s, truncating it.", height - 1)
|
||||
self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET)
|
||||
self.io.truncate()
|
||||
self.io.flush()
|
||||
self._size = self.io.seek(0, os.SEEK_END) // self.header_size
|
||||
self._size = None
|
||||
return
|
||||
previous_header_hash = header_hash
|
||||
|
||||
@@ -349,6 +276,10 @@ class Headers:
|
||||
header = headers[start:end]
|
||||
yield self.hash_header(header), self.deserialize(height+idx, header)
|
||||
|
||||
@property
|
||||
def claim_trie_root(self):
|
||||
return self[self.height]['claim_trie_root']
|
||||
|
||||
@staticmethod
|
||||
def header_hash_to_pow_hash(header_hash: bytes):
|
||||
header_hash_bytes = unhexlify(header_hash)[::-1]
|
||||
@@ -364,4 +295,3 @@ class UnvalidatedHeaders(Headers):
|
||||
validate_difficulty = False
|
||||
max_target = 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
|
||||
genesis_hash = b'6e3fcf1299d4ec5d79c3a4c91d624a4acf9e2e173d95a1a0504f677669687556'
|
||||
checkpoints = {}
|
||||
+196
-472
File diff suppressed because it is too large.
Load diff
+15
-18
@@ -184,7 +184,6 @@ class WalletManager:
|
||||
'auto_connect': True,
|
||||
'default_servers': config.lbryum_servers,
|
||||
'data_path': config.wallet_dir,
|
||||
'tx_cache_size': config.transaction_cache_size
|
||||
}
|
||||
|
||||
wallets_directory = os.path.join(config.wallet_dir, 'wallets')
|
||||
@@ -249,28 +248,26 @@ class WalletManager:
|
||||
log.warning("Failed to migrate %s receiving addresses!",
|
||||
len(set(receiving_addresses).difference(set(migrated_receiving))))
|
||||
|
||||
async def get_best_blockhash(self):
|
||||
def get_best_blockhash(self):
|
||||
if len(self.ledger.headers) <= 0:
|
||||
return self.ledger.genesis_hash
|
||||
return (await self.ledger.headers.hash(self.ledger.headers.height)).decode()
|
||||
return self.ledger.headers.hash(self.ledger.headers.height).decode()
|
||||
|
||||
def get_unused_address(self):
|
||||
return self.default_account.receiving.get_or_create_usable_address()
|
||||
|
||||
async def get_transaction(self, txid: str):
|
||||
async def get_transaction(self, txid):
|
||||
tx = await self.db.get_transaction(txid=txid)
|
||||
if tx:
|
||||
return tx
|
||||
try:
|
||||
raw, merkle = await self.ledger.network.get_transaction_and_merkle(txid)
|
||||
except CodeMessageError as e:
|
||||
if 'No such mempool or blockchain transaction.' in e.message:
|
||||
return {'success': False, 'code': 404, 'message': 'transaction not found'}
|
||||
return {'success': False, 'code': e.code, 'message': e.message}
|
||||
height = merkle.get('block_height')
|
||||
tx = Transaction(unhexlify(raw), height=height)
|
||||
if height and height > 0:
|
||||
await self.ledger.maybe_verify_transaction(tx, height, merkle)
|
||||
if not tx:
|
||||
try:
|
||||
raw = await self.ledger.network.get_transaction(txid)
|
||||
height = await self.ledger.network.get_transaction_height(txid)
|
||||
except CodeMessageError as e:
|
||||
if 'No such mempool or blockchain transaction.' in e.message:
|
||||
return {'success': False, 'code': 404, 'message': 'transaction not found'}
|
||||
return {'success': False, 'code': e.code, 'message': e.message}
|
||||
tx = Transaction(unhexlify(raw))
|
||||
await self.ledger.maybe_verify_transaction(tx, height)
|
||||
return tx
|
||||
|
||||
async def create_purchase_transaction(
|
||||
@@ -300,8 +297,8 @@ class WalletManager:
|
||||
async def broadcast_or_release(self, tx, blocking=False):
|
||||
try:
|
||||
await self.ledger.broadcast(tx)
|
||||
if blocking:
|
||||
await self.ledger.wait(tx, timeout=None)
|
||||
except:
|
||||
await self.ledger.release_tx(tx)
|
||||
raise
|
||||
if blocking:
|
||||
await self.ledger.wait(tx, timeout=None)
|
||||
+178
-235
@@ -1,28 +1,24 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import json
|
||||
import socket
|
||||
import random
|
||||
from time import perf_counter
|
||||
from collections import defaultdict
|
||||
from operator import itemgetter
|
||||
from typing import Dict, Optional, Tuple
|
||||
import aiohttp
|
||||
|
||||
from lbry import __version__
|
||||
from lbry.utils import resolve_host
|
||||
from lbry.error import IncompatibleWalletServerError
|
||||
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
|
||||
from lbry.wallet.stream import StreamController
|
||||
from lbry.wallet.server.udp import SPVStatusClientProtocol, SPVPong
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClientSession(BaseClientSession):
|
||||
def __init__(self, *args, network: 'Network', server, timeout=30, **kwargs):
|
||||
def __init__(self, *args, network, server, timeout=30, on_connect_callback=None, **kwargs):
|
||||
self.network = network
|
||||
self.server = server
|
||||
super().__init__(*args, **kwargs)
|
||||
self._on_disconnect_controller = StreamController()
|
||||
self.on_disconnected = self._on_disconnect_controller.stream
|
||||
self.framer.max_size = self.max_errors = 1 << 32
|
||||
self.timeout = timeout
|
||||
self.max_seconds_idle = timeout * 2
|
||||
@@ -30,6 +26,8 @@ class ClientSession(BaseClientSession):
|
||||
self.connection_latency: Optional[float] = None
|
||||
self._response_samples = 0
|
||||
self.pending_amount = 0
|
||||
self._on_connect_cb = on_connect_callback or (lambda: None)
|
||||
self.trigger_urgent_reconnect = asyncio.Event()
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
@@ -56,7 +54,7 @@ class ClientSession(BaseClientSession):
|
||||
|
||||
async def send_request(self, method, args=()):
|
||||
self.pending_amount += 1
|
||||
log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout)
|
||||
log.debug("send %s to %s:%i", method, *self.server)
|
||||
try:
|
||||
if method == 'server.version':
|
||||
return await self.send_timed_server_version_request(args, self.timeout)
|
||||
@@ -67,17 +65,10 @@ class ClientSession(BaseClientSession):
|
||||
log.debug("Time since last packet: %s", perf_counter() - self.last_packet_received)
|
||||
if (perf_counter() - self.last_packet_received) < self.timeout:
|
||||
continue
|
||||
log.warning("timeout sending %s to %s:%i", method, *self.server)
|
||||
log.info("timeout sending %s to %s:%i", method, *self.server)
|
||||
raise asyncio.TimeoutError
|
||||
if done:
|
||||
try:
|
||||
return request.result()
|
||||
except ConnectionResetError:
|
||||
log.error(
|
||||
"wallet server (%s) reset connection upon our %s request, json of %i args is %i bytes",
|
||||
self.server[0], method, len(args), len(json.dumps(args))
|
||||
)
|
||||
raise
|
||||
return request.result()
|
||||
except (RPCError, ProtocolError) as e:
|
||||
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
|
||||
*self.server, *e.args)
|
||||
@@ -87,12 +78,44 @@ class ClientSession(BaseClientSession):
|
||||
self.synchronous_close()
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
log.warning("cancelled sending %s to %s:%i", method, *self.server)
|
||||
# self.synchronous_close()
|
||||
log.info("cancelled sending %s to %s:%i", method, *self.server)
|
||||
self.synchronous_close()
|
||||
raise
|
||||
finally:
|
||||
self.pending_amount -= 1
|
||||
|
||||
async def ensure_session(self):
|
||||
# Handles reconnecting and maintaining a session alive
|
||||
# TODO: change to 'ping' on newer protocol (above 1.2)
|
||||
retry_delay = default_delay = 1.0
|
||||
while True:
|
||||
try:
|
||||
if self.is_closing():
|
||||
await self.create_connection(self.timeout)
|
||||
await self.ensure_server_version()
|
||||
self._on_connect_cb()
|
||||
elif (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
|
||||
await self.ensure_server_version()
|
||||
retry_delay = default_delay
|
||||
except RPCError as e:
|
||||
await self.close()
|
||||
log.debug("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
|
||||
retry_delay = 60 * 60
|
||||
except IncompatibleWalletServerError:
|
||||
await self.close()
|
||||
retry_delay = 60 * 60
|
||||
log.debug("Wallet server has an incompatible version, retrying in 1h: %s:%d", *self.server)
|
||||
except (asyncio.TimeoutError, OSError):
|
||||
await self.close()
|
||||
retry_delay = min(60, retry_delay * 2)
|
||||
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
|
||||
try:
|
||||
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
finally:
|
||||
self.trigger_urgent_reconnect.clear()
|
||||
|
||||
async def ensure_server_version(self, required=None, timeout=3):
|
||||
required = required or self.network.PROTOCOL_VERSION
|
||||
response = await asyncio.wait_for(
|
||||
@@ -102,25 +125,6 @@ class ClientSession(BaseClientSession):
|
||||
raise IncompatibleWalletServerError(*self.server)
|
||||
return response
|
||||
|
||||
async def keepalive_loop(self, timeout=3, max_idle=60):
|
||||
try:
|
||||
while True:
|
||||
now = perf_counter()
|
||||
if min(self.last_send, self.last_packet_received) + max_idle < now:
|
||||
await asyncio.wait_for(
|
||||
self.send_request('server.ping', []), timeout=timeout
|
||||
)
|
||||
else:
|
||||
await asyncio.sleep(max(0, max_idle - (now - self.last_send)))
|
||||
except Exception as err:
|
||||
if isinstance(err, asyncio.CancelledError):
|
||||
log.warning("closing connection to %s:%i", *self.server)
|
||||
else:
|
||||
log.exception("lost connection to spv")
|
||||
finally:
|
||||
if not self.is_closing():
|
||||
self._close()
|
||||
|
||||
async def create_connection(self, timeout=6):
|
||||
connector = Connector(lambda: self, *self.server)
|
||||
start = perf_counter()
|
||||
@@ -132,26 +136,25 @@ class ClientSession(BaseClientSession):
|
||||
controller.add(request.args)
|
||||
|
||||
def connection_lost(self, exc):
|
||||
log.warning("Connection lost: %s:%d", *self.server)
|
||||
log.debug("Connection lost: %s:%d", *self.server)
|
||||
super().connection_lost(exc)
|
||||
self.response_time = None
|
||||
self.connection_latency = None
|
||||
self._response_samples = 0
|
||||
# self._on_disconnect_controller.add(True)
|
||||
if self.network:
|
||||
self.network.disconnect()
|
||||
self.pending_amount = 0
|
||||
self._on_disconnect_controller.add(True)
|
||||
|
||||
|
||||
class Network:
|
||||
|
||||
PROTOCOL_VERSION = __version__
|
||||
MINIMUM_REQUIRED = (0, 65, 0)
|
||||
MINIMUM_REQUIRED = (0, 54, 0)
|
||||
|
||||
def __init__(self, ledger):
|
||||
self.ledger = ledger
|
||||
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
|
||||
self.client: Optional[ClientSession] = None
|
||||
self.server_features = None
|
||||
# self._switch_task: Optional[asyncio.Task] = None
|
||||
self._switch_task: Optional[asyncio.Task] = None
|
||||
self.running = False
|
||||
self.remote_height: int = 0
|
||||
self._concurrency = asyncio.Semaphore(16)
|
||||
@@ -170,177 +173,52 @@ class Network:
|
||||
'blockchain.address.subscribe': self._on_status_controller,
|
||||
}
|
||||
|
||||
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
|
||||
self._urgent_need_reconnect = asyncio.Event()
|
||||
self._loop_task: Optional[asyncio.Task] = None
|
||||
self._keepalive_task: Optional[asyncio.Task] = None
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
return self.ledger.config
|
||||
|
||||
def disconnect(self):
|
||||
if self._keepalive_task and not self._keepalive_task.done():
|
||||
self._keepalive_task.cancel()
|
||||
self._keepalive_task = None
|
||||
|
||||
async def start(self):
|
||||
if not self.running:
|
||||
self.running = True
|
||||
self.aiohttp_session = aiohttp.ClientSession()
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
self._loop_task = asyncio.create_task(self.network_loop())
|
||||
self._urgent_need_reconnect.set()
|
||||
|
||||
def loop_task_done_callback(f):
|
||||
try:
|
||||
f.result()
|
||||
except Exception:
|
||||
if self.running:
|
||||
log.exception("wallet server connection loop crashed")
|
||||
|
||||
self._loop_task.add_done_callback(loop_task_done_callback)
|
||||
|
||||
async def resolve_spv_dns(self):
|
||||
hostname_to_ip = {}
|
||||
ip_to_hostnames = defaultdict(list)
|
||||
|
||||
async def resolve_spv(server, port):
|
||||
try:
|
||||
server_addr = await resolve_host(server, port, 'udp')
|
||||
hostname_to_ip[server] = (server_addr, port)
|
||||
ip_to_hostnames[(server_addr, port)].append(server)
|
||||
except socket.error:
|
||||
log.warning("error looking up dns for spv server %s:%i", server, port)
|
||||
except Exception:
|
||||
log.exception("error looking up dns for spv server %s:%i", server, port)
|
||||
|
||||
# accumulate the dns results
|
||||
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in self.config['default_servers']))
|
||||
return hostname_to_ip, ip_to_hostnames
|
||||
|
||||
async def get_n_fastest_spvs(self, timeout=3.0) -> Dict[Tuple[str, int], Optional[SPVPong]]:
|
||||
loop = asyncio.get_event_loop()
|
||||
pong_responses = asyncio.Queue()
|
||||
connection = SPVStatusClientProtocol(pong_responses)
|
||||
sent_ping_timestamps = {}
|
||||
_, ip_to_hostnames = await self.resolve_spv_dns()
|
||||
n = len(ip_to_hostnames)
|
||||
log.info("%i possible spv servers to try (%i urls in config)", n, len(self.config['default_servers']))
|
||||
pongs = {}
|
||||
try:
|
||||
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
|
||||
# could raise OSError if it cant bind
|
||||
start = perf_counter()
|
||||
for server in ip_to_hostnames:
|
||||
connection.ping(server)
|
||||
sent_ping_timestamps[server] = perf_counter()
|
||||
while len(pongs) < n:
|
||||
(remote, ts), pong = await asyncio.wait_for(pong_responses.get(), timeout - (perf_counter() - start))
|
||||
latency = ts - start
|
||||
log.info("%s:%i has latency of %sms (available: %s, height: %i)",
|
||||
'/'.join(ip_to_hostnames[remote]), remote[1], round(latency * 1000, 2),
|
||||
pong.available, pong.height)
|
||||
|
||||
if pong.available:
|
||||
pongs[(ip_to_hostnames[remote][0], remote[1])] = pong
|
||||
return pongs
|
||||
except asyncio.TimeoutError:
|
||||
if pongs:
|
||||
log.info("%i/%i probed spv servers are accepting connections", len(pongs), len(ip_to_hostnames))
|
||||
return pongs
|
||||
else:
|
||||
log.warning("%i spv status probes failed, retrying later. servers tried: %s",
|
||||
len(sent_ping_timestamps),
|
||||
', '.join('/'.join(hosts) + f' ({ip})' for ip, hosts in ip_to_hostnames.items()))
|
||||
random_server = random.choice(list(ip_to_hostnames.keys()))
|
||||
host, port = random_server
|
||||
log.warning("trying fallback to randomly selected spv: %s:%i", host, port)
|
||||
return {(host, port): None}
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
async def connect_to_fastest(self) -> Optional[ClientSession]:
|
||||
fastest_spvs = await self.get_n_fastest_spvs()
|
||||
for (host, port) in fastest_spvs:
|
||||
client = ClientSession(network=self, server=(host, port))
|
||||
try:
|
||||
await client.create_connection()
|
||||
log.warning("Connected to spv server %s:%i", host, port)
|
||||
await client.ensure_server_version()
|
||||
return client
|
||||
except (asyncio.TimeoutError, ConnectionError, OSError, IncompatibleWalletServerError, RPCError):
|
||||
log.warning("Connecting to %s:%d failed", host, port)
|
||||
client._close()
|
||||
return
|
||||
|
||||
async def network_loop(self):
|
||||
sleep_delay = 30
|
||||
async def switch_forever(self):
|
||||
while self.running:
|
||||
await asyncio.wait(
|
||||
[asyncio.sleep(30), self._urgent_need_reconnect.wait()], return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
if self._urgent_need_reconnect.is_set():
|
||||
sleep_delay = 30
|
||||
self._urgent_need_reconnect.clear()
|
||||
if not self.is_connected:
|
||||
client = await self.connect_to_fastest()
|
||||
if not client:
|
||||
log.warning("failed to connect to any spv servers, retrying later")
|
||||
sleep_delay *= 2
|
||||
sleep_delay = min(sleep_delay, 300)
|
||||
continue
|
||||
log.debug("get spv server features %s:%i", *client.server)
|
||||
features = await client.send_request('server.features', [])
|
||||
self.client, self.server_features = client, features
|
||||
log.info("subscribe to headers %s:%i", *client.server)
|
||||
if self.is_connected:
|
||||
await self.client.on_disconnected.first
|
||||
self.client = None
|
||||
continue
|
||||
self.client = await self.session_pool.wait_for_fastest_session()
|
||||
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
|
||||
try:
|
||||
self._update_remote_height((await self.subscribe_headers(),))
|
||||
self._on_connected_controller.add(True)
|
||||
server_str = "%s:%i" % client.server
|
||||
log.info("maintaining connection to spv server %s", server_str)
|
||||
self._keepalive_task = asyncio.create_task(self.client.keepalive_loop())
|
||||
try:
|
||||
if not self._urgent_need_reconnect.is_set():
|
||||
await asyncio.wait(
|
||||
[self._keepalive_task, self._urgent_need_reconnect.wait()],
|
||||
return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
else:
|
||||
await self._keepalive_task
|
||||
if self._urgent_need_reconnect.is_set():
|
||||
log.warning("urgent reconnect needed")
|
||||
self._urgent_need_reconnect.clear()
|
||||
if self._keepalive_task and not self._keepalive_task.done():
|
||||
self._keepalive_task.cancel()
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
self._keepalive_task = None
|
||||
self.client = None
|
||||
self.server_features = None
|
||||
log.warning("connection lost to %s", server_str)
|
||||
log.info("network loop finished")
|
||||
log.info("Subscribed to headers: %s:%d", *self.client.server)
|
||||
except (asyncio.TimeoutError, ConnectionError):
|
||||
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
|
||||
self.client.synchronous_close()
|
||||
self.client = None
|
||||
|
||||
async def start(self):
|
||||
self.running = True
|
||||
self._switch_task = asyncio.ensure_future(self.switch_forever())
|
||||
# this may become unnecessary when there are no more bugs found,
|
||||
# but for now it helps understanding log reports
|
||||
self._switch_task.add_done_callback(lambda _: log.info("Wallet client switching task stopped."))
|
||||
self.session_pool.start(self.config['default_servers'])
|
||||
self.on_header.listen(self._update_remote_height)
|
||||
|
||||
async def stop(self):
|
||||
self.running = False
|
||||
self.disconnect()
|
||||
if self._loop_task and not self._loop_task.done():
|
||||
self._loop_task.cancel()
|
||||
self._loop_task = None
|
||||
if self.aiohttp_session:
|
||||
await self.aiohttp_session.close()
|
||||
self.aiohttp_session = None
|
||||
if self.running:
|
||||
self.running = False
|
||||
self._switch_task.cancel()
|
||||
self.session_pool.stop()
|
||||
|
||||
@property
|
||||
def is_connected(self):
|
||||
return self.client and not self.client.is_closing()
|
||||
|
||||
def rpc(self, list_or_method, args, restricted=True, session: Optional[ClientSession] = None):
|
||||
if session or self.is_connected:
|
||||
session = session or self.client
|
||||
def rpc(self, list_or_method, args, restricted=True):
|
||||
session = self.client if restricted else self.session_pool.fastest_session
|
||||
if session and not session.is_closing():
|
||||
return session.send_request(list_or_method, args)
|
||||
else:
|
||||
self._urgent_need_reconnect.set()
|
||||
self.session_pool.trigger_nodelay_connect()
|
||||
raise ConnectionError("Attempting to send rpc request when connection is not available.")
|
||||
|
||||
async def retriable_call(self, function, *args, **kwargs):
|
||||
@@ -348,15 +226,14 @@ class Network:
|
||||
while self.running:
|
||||
if not self.is_connected:
|
||||
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
|
||||
self._urgent_need_reconnect.set()
|
||||
await self.on_connected.first
|
||||
await self.session_pool.wait_for_fastest_session()
|
||||
try:
|
||||
return await function(*args, **kwargs)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("Wallet server call timed out, retrying.")
|
||||
except ConnectionError:
|
||||
log.warning("connection error")
|
||||
|
||||
pass
|
||||
raise asyncio.CancelledError() # if we got here, we are shutting down
|
||||
|
||||
def _update_remote_height(self, header_args):
|
||||
@@ -367,15 +244,6 @@ class Network:
|
||||
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
|
||||
return self.rpc('blockchain.transaction.get', [tx_hash], restricted)
|
||||
|
||||
def get_transaction_batch(self, txids, restricted=True):
|
||||
# use any server if its old, otherwise restrict to who gave us the history
|
||||
return self.rpc('blockchain.transaction.get_batch', txids, restricted)
|
||||
|
||||
def get_transaction_and_merkle(self, tx_hash, known_height=None):
|
||||
# use any server if its old, otherwise restrict to who gave us the history
|
||||
restricted = known_height in (None, -1, 0) or 0 > known_height > self.remote_height - 10
|
||||
return self.rpc('blockchain.transaction.info', [tx_hash], restricted)
|
||||
|
||||
def get_transaction_height(self, tx_hash, known_height=None):
|
||||
restricted = not known_height or 0 > known_height > self.remote_height - 10
|
||||
return self.rpc('blockchain.transaction.get_height', [tx_hash], restricted)
|
||||
@@ -400,13 +268,12 @@ class Network:
|
||||
|
||||
async def subscribe_address(self, address, *addresses):
|
||||
addresses = list((address, ) + addresses)
|
||||
server_addr_and_port = self.client.server_address_and_port # on disconnect client will be None
|
||||
try:
|
||||
return await self.rpc('blockchain.address.subscribe', addresses, True)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning(
|
||||
"timed out subscribing to addresses from %s:%i",
|
||||
*server_addr_and_port
|
||||
*self.client.server_address_and_port
|
||||
)
|
||||
# abort and cancel, we can't lose a subscription, it will happen again on reconnect
|
||||
if self.client:
|
||||
@@ -419,27 +286,103 @@ class Network:
|
||||
def get_server_features(self):
|
||||
return self.rpc('server.features', (), restricted=True)
|
||||
|
||||
def resolve(self, urls, session_override=None):
|
||||
return self.rpc('blockchain.claimtrie.resolve', urls, False, session_override)
|
||||
def get_claims_by_ids(self, claim_ids):
|
||||
return self.rpc('blockchain.claimtrie.getclaimsbyids', claim_ids)
|
||||
|
||||
def claim_search(self, session_override=None, **kwargs):
|
||||
return self.rpc('blockchain.claimtrie.search', kwargs, False, session_override)
|
||||
def resolve(self, urls):
|
||||
return self.rpc('blockchain.claimtrie.resolve', urls)
|
||||
|
||||
async def new_resolve(self, server, urls):
|
||||
message = {"method": "resolve", "params": {"urls": urls, "protobuf": True}}
|
||||
async with self.aiohttp_session.post(server, json=message) as r:
|
||||
result = await r.json()
|
||||
return result['result']
|
||||
def claim_search(self, **kwargs):
|
||||
return self.rpc('blockchain.claimtrie.search', kwargs)
|
||||
|
||||
async def new_claim_search(self, server, **kwargs):
|
||||
kwargs['protobuf'] = True
|
||||
message = {"method": "claim_search", "params": kwargs}
|
||||
async with self.aiohttp_session.post(server, json=message) as r:
|
||||
result = await r.json()
|
||||
return result['result']
|
||||
|
||||
async def sum_supports(self, server, **kwargs):
|
||||
message = {"method": "support_sum", "params": kwargs}
|
||||
async with self.aiohttp_session.post(server, json=message) as r:
|
||||
result = await r.json()
|
||||
return result['result']
|
||||
class SessionPool:
|
||||
|
||||
def __init__(self, network: Network, timeout: float):
|
||||
self.network = network
|
||||
self.sessions: Dict[ClientSession, Optional[asyncio.Task]] = dict()
|
||||
self.timeout = timeout
|
||||
self.new_connection_event = asyncio.Event()
|
||||
|
||||
@property
|
||||
def online(self):
|
||||
return any(not session.is_closing() for session in self.sessions)
|
||||
|
||||
@property
|
||||
def available_sessions(self):
|
||||
return (session for session in self.sessions if session.available)
|
||||
|
||||
@property
|
||||
def fastest_session(self):
|
||||
if not self.online:
|
||||
return None
|
||||
return min(
|
||||
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
|
||||
for session in self.available_sessions] or [(0, None)],
|
||||
key=itemgetter(0)
|
||||
)[1]
|
||||
|
||||
def _get_session_connect_callback(self, session: ClientSession):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def callback():
|
||||
duplicate_connections = [
|
||||
s for s in self.sessions
|
||||
if s is not session and s.server_address_and_port == session.server_address_and_port
|
||||
]
|
||||
already_connected = None if not duplicate_connections else duplicate_connections[0]
|
||||
if already_connected:
|
||||
self.sessions.pop(session).cancel()
|
||||
session.synchronous_close()
|
||||
log.debug("wallet server %s resolves to the same server as %s, rechecking in an hour",
|
||||
session.server[0], already_connected.server[0])
|
||||
loop.call_later(3600, self._connect_session, session.server)
|
||||
return
|
||||
self.new_connection_event.set()
|
||||
log.info("connected to %s:%i", *session.server)
|
||||
|
||||
return callback
|
||||
|
||||
def _connect_session(self, server: Tuple[str, int]):
|
||||
session = None
|
||||
for s in self.sessions:
|
||||
if s.server == server:
|
||||
session = s
|
||||
break
|
||||
if not session:
|
||||
session = ClientSession(
|
||||
network=self.network, server=server
|
||||
)
|
||||
session._on_connect_cb = self._get_session_connect_callback(session)
|
||||
task = self.sessions.get(session, None)
|
||||
if not task or task.done():
|
||||
task = asyncio.create_task(session.ensure_session())
|
||||
task.add_done_callback(lambda _: self.ensure_connections())
|
||||
self.sessions[session] = task
|
||||
|
||||
def start(self, default_servers):
|
||||
for server in default_servers:
|
||||
self._connect_session(server)
|
||||
|
||||
def stop(self):
|
||||
for session, task in self.sessions.items():
|
||||
task.cancel()
|
||||
session.synchronous_close()
|
||||
self.sessions.clear()
|
||||
|
||||
def ensure_connections(self):
|
||||
for session in self.sessions:
|
||||
self._connect_session(session.server)
|
||||
|
||||
def trigger_nodelay_connect(self):
|
||||
# used when other parts of the system sees we might have internet back
|
||||
# bypasses the retry interval
|
||||
for session in self.sessions:
|
||||
session.trigger_urgent_reconnect.set()
|
||||
|
||||
async def wait_for_fastest_session(self):
|
||||
while not self.fastest_session:
|
||||
self.trigger_nodelay_connect()
|
||||
self.new_connection_event.clear()
|
||||
await self.new_connection_event.wait()
|
||||
return self.fastest_session
|
||||
@@ -11,7 +11,6 @@ import importlib
|
||||
from binascii import hexlify
|
||||
from typing import Type, Optional
|
||||
import urllib.request
|
||||
from uuid import uuid4
|
||||
|
||||
import lbry
|
||||
from lbry.wallet.server.server import Server
|
||||
@@ -56,8 +55,7 @@ class Conductor:
|
||||
|
||||
async def start_blockchain(self):
|
||||
if not self.blockchain_started:
|
||||
asyncio.create_task(self.blockchain_node.start())
|
||||
await self.blockchain_node.running.wait()
|
||||
await self.blockchain_node.start()
|
||||
await self.blockchain_node.generate(200)
|
||||
self.blockchain_started = True
|
||||
|
||||
@@ -169,28 +167,21 @@ class SPVNode:
|
||||
self.server = None
|
||||
self.hostname = 'localhost'
|
||||
self.port = 50001 + node_number # avoid conflict with default daemon
|
||||
self.udp_port = self.port
|
||||
self.session_timeout = 600
|
||||
self.rpc_port = '0' # disabled by default
|
||||
|
||||
async def start(self, blockchain_node: 'BlockchainNode', extraconf=None):
|
||||
self.data_path = tempfile.mkdtemp()
|
||||
conf = {
|
||||
'DESCRIPTION': '',
|
||||
'PAYMENT_ADDRESS': '',
|
||||
'DAILY_FEE': '0',
|
||||
'DB_DIRECTORY': self.data_path,
|
||||
'DAEMON_URL': blockchain_node.rpc_url,
|
||||
'REORG_LIMIT': '100',
|
||||
'HOST': self.hostname,
|
||||
'TCP_PORT': str(self.port),
|
||||
'UDP_PORT': str(self.udp_port),
|
||||
'SESSION_TIMEOUT': str(self.session_timeout),
|
||||
'MAX_QUERY_WORKERS': '0',
|
||||
'INDIVIDUAL_TAG_INDEXES': '',
|
||||
'RPC_PORT': self.rpc_port,
|
||||
'ES_INDEX_PREFIX': uuid4().hex,
|
||||
'ES_MODE': 'writer',
|
||||
'RPC_PORT': self.rpc_port
|
||||
}
|
||||
if extraconf:
|
||||
conf.update(extraconf)
|
||||
@@ -202,8 +193,6 @@ class SPVNode:
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
try:
|
||||
await self.server.db.search_index.delete_index()
|
||||
await self.server.db.search_index.stop()
|
||||
await self.server.stop()
|
||||
finally:
|
||||
cleanup and self.cleanup()
|
||||
@@ -239,7 +228,6 @@ class BlockchainProcess(asyncio.SubprocessProtocol):
|
||||
|
||||
def process_exited(self):
|
||||
self.stopped.set()
|
||||
self.ready.set()
|
||||
|
||||
|
||||
class BlockchainNode:
|
||||
@@ -263,10 +251,6 @@ class BlockchainNode:
|
||||
self.rpcport = 9245 + 2 # avoid conflict with default rpc port
|
||||
self.rpcuser = 'rpcuser'
|
||||
self.rpcpassword = 'rpcpassword'
|
||||
self.stopped = False
|
||||
self.restart_ready = asyncio.Event()
|
||||
self.restart_ready.set()
|
||||
self.running = asyncio.Event()
|
||||
|
||||
@property
|
||||
def rpc_url(self):
|
||||
@@ -327,27 +311,12 @@ class BlockchainNode:
|
||||
f'-port={self.peerport}'
|
||||
]
|
||||
self.log.info(' '.join(command))
|
||||
while not self.stopped:
|
||||
if self.running.is_set():
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
await self.restart_ready.wait()
|
||||
try:
|
||||
self.transport, self.protocol = await loop.subprocess_exec(
|
||||
BlockchainProcess, *command
|
||||
)
|
||||
await self.protocol.ready.wait()
|
||||
assert not self.protocol.stopped.is_set()
|
||||
self.running.set()
|
||||
except asyncio.CancelledError:
|
||||
self.running.clear()
|
||||
raise
|
||||
except Exception as e:
|
||||
self.running.clear()
|
||||
log.exception('failed to start lbrycrdd', exc_info=e)
|
||||
self.transport, self.protocol = await loop.subprocess_exec(
|
||||
BlockchainProcess, *command
|
||||
)
|
||||
await self.protocol.ready.wait()
|
||||
|
||||
async def stop(self, cleanup=True):
|
||||
self.stopped = True
|
||||
try:
|
||||
self.transport.terminate()
|
||||
await self.protocol.stopped.wait()
|
||||
@@ -356,16 +325,6 @@ class BlockchainNode:
|
||||
if cleanup:
|
||||
self.cleanup()
|
||||
|
||||
async def clear_mempool(self):
|
||||
self.restart_ready.clear()
|
||||
self.transport.terminate()
|
||||
await self.protocol.stopped.wait()
|
||||
self.transport.close()
|
||||
self.running.clear()
|
||||
os.remove(os.path.join(self.data_path, 'regtest', 'mempool.dat'))
|
||||
self.restart_ready.set()
|
||||
await self.running.wait()
|
||||
|
||||
def cleanup(self):
|
||||
shutil.rmtree(self.data_path, ignore_errors=True)
|
||||
|
||||
@@ -397,12 +356,6 @@ class BlockchainNode:
|
||||
def get_block_hash(self, block):
|
||||
return self._cli_cmnd('getblockhash', str(block))
|
||||
|
||||
def sendrawtransaction(self, tx):
|
||||
return self._cli_cmnd('sendrawtransaction', tx)
|
||||
|
||||
async def get_block(self, block_hash):
|
||||
return json.loads(await self._cli_cmnd('getblock', block_hash, '1'))
|
||||
|
||||
def get_raw_change_address(self):
|
||||
return self._cli_cmnd('getrawchangeaddress')
|
||||
|
||||
@@ -410,7 +363,7 @@ class BlockchainNode:
|
||||
return self._cli_cmnd('getnewaddress', "", address_type)
|
||||
|
||||
async def get_balance(self):
|
||||
return await self._cli_cmnd('getbalance')
|
||||
return float(await self._cli_cmnd('getbalance'))
|
||||
|
||||
def send_to_address(self, address, amount):
|
||||
return self._cli_cmnd('sendtoaddress', address, str(amount))
|
||||
|
||||
@@ -108,6 +108,9 @@ class Response:
|
||||
|
||||
class CodeMessageError(Exception):
|
||||
|
||||
def __init__(self, code, message):
|
||||
super().__init__(code, message)
|
||||
|
||||
@property
|
||||
def code(self):
|
||||
return self.args[0]
|
||||
|
||||
@@ -33,16 +33,13 @@ from asyncio import Event, CancelledError
|
||||
import logging
|
||||
import time
|
||||
from contextlib import suppress
|
||||
from prometheus_client import Counter, Histogram
|
||||
|
||||
from lbry.wallet.tasks import TaskGroup
|
||||
|
||||
from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification
|
||||
from .jsonrpc import RPCError, ProtocolError
|
||||
from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer
|
||||
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
from .util import Concurrency
|
||||
|
||||
|
||||
class Connector:
|
||||
@@ -206,7 +203,7 @@ class SessionBase(asyncio.Protocol):
|
||||
"""
|
||||
return self._address
|
||||
|
||||
def peer_address_str(self, for_log=True):
|
||||
def peer_address_str(self):
|
||||
"""Returns the peer's IP address and port as a human-readable
|
||||
string."""
|
||||
if not self._address:
|
||||
@@ -375,41 +372,21 @@ class BatchRequest:
|
||||
raise BatchError(self)
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
|
||||
|
||||
class RPCSession(SessionBase):
|
||||
"""Base class for protocols where a message can lead to a response,
|
||||
for example JSON RPC."""
|
||||
|
||||
RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE,
|
||||
labelnames=("method", "version"), buckets=HISTOGRAM_BUCKETS)
|
||||
NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)",
|
||||
namespace=NAMESPACE, labelnames=("method", "version"))
|
||||
REQUEST_ERRORS_COUNT = Counter(
|
||||
"request_error", "Number of requests that returned errors", namespace=NAMESPACE,
|
||||
labelnames=("method", "version")
|
||||
)
|
||||
RESET_CONNECTIONS = Counter(
|
||||
"reset_clients", "Number of reset connections by client version",
|
||||
namespace=NAMESPACE, labelnames=("version",)
|
||||
)
|
||||
|
||||
def __init__(self, *, framer=None, loop=None, connection=None):
|
||||
super().__init__(framer=framer, loop=loop)
|
||||
self.connection = connection or self.default_connection()
|
||||
self.client_version = 'unknown'
|
||||
|
||||
async def _receive_messages(self):
|
||||
while not self.is_closing():
|
||||
try:
|
||||
message = await self.framer.receive_message()
|
||||
except MemoryError:
|
||||
self.logger.warning('received oversized message from %s:%s, dropping connection',
|
||||
self._address[0], self._address[1])
|
||||
self.RESET_CONNECTIONS.labels(version=self.client_version).inc()
|
||||
self._close()
|
||||
return
|
||||
except MemoryError as e:
|
||||
self.logger.warning(f'{e!r}')
|
||||
continue
|
||||
|
||||
self.last_recv = time.perf_counter()
|
||||
self.recv_count += 1
|
||||
@@ -428,7 +405,6 @@ class RPCSession(SessionBase):
|
||||
await self._task_group.add(self._handle_request(request))
|
||||
|
||||
async def _handle_request(self, request):
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
result = await self.handle_request(request)
|
||||
except (ProtocolError, RPCError) as e:
|
||||
@@ -441,18 +417,10 @@ class RPCSession(SessionBase):
|
||||
'internal server error')
|
||||
if isinstance(request, Request):
|
||||
message = request.send_result(result)
|
||||
self.RESPONSE_TIMES.labels(
|
||||
method=request.method,
|
||||
version=self.client_version
|
||||
).observe(time.perf_counter() - start)
|
||||
if message:
|
||||
await self._send_message(message)
|
||||
if isinstance(result, Exception):
|
||||
self._bump_errors()
|
||||
self.REQUEST_ERRORS_COUNT.labels(
|
||||
method=request.method,
|
||||
version=self.client_version
|
||||
).inc()
|
||||
|
||||
def connection_lost(self, exc):
|
||||
# Cancel pending requests and message processing
|
||||
@@ -483,17 +451,10 @@ class RPCSession(SessionBase):
|
||||
raise result
|
||||
return result
|
||||
|
||||
async def send_notification(self, method, args=()) -> bool:
|
||||
async def send_notification(self, method, args=()):
|
||||
"""Send an RPC notification over the network."""
|
||||
message = self.connection.send_notification(Notification(method, args))
|
||||
self.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc()
|
||||
try:
|
||||
await self._send_message(message)
|
||||
return True
|
||||
except asyncio.TimeoutError:
|
||||
self.logger.info("timeout sending address notification to %s", self.peer_address_str(for_log=True))
|
||||
self.abort()
|
||||
return False
|
||||
await self._send_message(message)
|
||||
|
||||
def send_batch(self, raise_errors=False):
|
||||
"""Return a BatchRequest. Intended to be used like so:
|
||||
|
||||
@@ -32,13 +32,10 @@ import inspect
|
||||
|
||||
# other_params: None means cannot be called with keyword arguments only
|
||||
# any means any name is good
|
||||
from functools import lru_cache
|
||||
|
||||
SignatureInfo = namedtuple('SignatureInfo', 'min_args max_args '
|
||||
'required_names other_names')
|
||||
|
||||
|
||||
@lru_cache(256)
|
||||
def signature_info(func):
|
||||
params = inspect.signature(func).parameters
|
||||
min_args = max_args = 0
|
||||
|
||||
@@ -438,17 +438,6 @@ class OutputScript(Script):
|
||||
SUPPORT_CLAIM_OPCODES + PAY_SCRIPT_HASH.opcodes
|
||||
))
|
||||
|
||||
SUPPORT_CLAIM_DATA_OPCODES = (
|
||||
OP_SUPPORT_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('support'),
|
||||
OP_2DROP, OP_2DROP
|
||||
)
|
||||
SUPPORT_CLAIM_DATA_PUBKEY = Template('support_claim+data+pay_pubkey_hash', (
|
||||
SUPPORT_CLAIM_DATA_OPCODES + PAY_PUBKEY_HASH.opcodes
|
||||
))
|
||||
SUPPORT_CLAIM_DATA_SCRIPT = Template('support_claim+data+pay_script_hash', (
|
||||
SUPPORT_CLAIM_DATA_OPCODES + PAY_SCRIPT_HASH.opcodes
|
||||
))
|
||||
|
||||
UPDATE_CLAIM_OPCODES = (
|
||||
OP_UPDATE_CLAIM, PUSH_SINGLE('claim_name'), PUSH_SINGLE('claim_id'), PUSH_SINGLE('claim'),
|
||||
OP_2DROP, OP_2DROP
|
||||
@@ -485,8 +474,6 @@ class OutputScript(Script):
|
||||
CLAIM_NAME_SCRIPT,
|
||||
SUPPORT_CLAIM_PUBKEY,
|
||||
SUPPORT_CLAIM_SCRIPT,
|
||||
SUPPORT_CLAIM_DATA_PUBKEY,
|
||||
SUPPORT_CLAIM_DATA_SCRIPT,
|
||||
UPDATE_CLAIM_PUBKEY,
|
||||
UPDATE_CLAIM_SCRIPT,
|
||||
SELL_CLAIM, SELL_SCRIPT,
|
||||
@@ -540,16 +527,6 @@ class OutputScript(Script):
|
||||
'pubkey_hash': pubkey_hash
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def pay_support_data_pubkey_hash(
|
||||
cls, claim_name: bytes, claim_id: bytes, support, pubkey_hash: bytes):
|
||||
return cls(template=cls.SUPPORT_CLAIM_DATA_PUBKEY, values={
|
||||
'claim_name': claim_name,
|
||||
'claim_id': claim_id,
|
||||
'support': support,
|
||||
'pubkey_hash': pubkey_hash
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def sell_script(cls, price):
|
||||
return cls(template=cls.SELL_SCRIPT, values={
|
||||
@@ -598,10 +575,6 @@ class OutputScript(Script):
|
||||
def is_support_claim(self):
|
||||
return self.template.name.startswith('support_claim+')
|
||||
|
||||
@property
|
||||
def is_support_claim_data(self):
|
||||
return self.template.name.startswith('support_claim+data+')
|
||||
|
||||
@property
|
||||
def is_sell_claim(self):
|
||||
return self.template.name.startswith('sell_claim+')
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
import time
|
||||
import asyncio
|
||||
from struct import pack, unpack
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
from typing import Optional
|
||||
from prometheus_client import Gauge, Histogram
|
||||
|
||||
import lbry
|
||||
from lbry.schema.claim import Claim
|
||||
from lbry.wallet.server.db.writer import SQLDB
|
||||
from lbry.wallet.server.daemon import DaemonError
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.util import chunks, class_logger
|
||||
from lbry.wallet.server.leveldb import FlushData
|
||||
from lbry.wallet.server.udp import StatusServer
|
||||
|
||||
|
||||
class Prefetcher:
|
||||
@@ -129,12 +127,6 @@ class ChainError(Exception):
|
||||
"""Raised on error processing blocks."""
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
|
||||
|
||||
class BlockProcessor:
|
||||
"""Process blocks and update the DB state to match.
|
||||
|
||||
@@ -142,16 +134,6 @@ class BlockProcessor:
|
||||
Coordinate backing up in case of chain reorganisations.
|
||||
"""
|
||||
|
||||
block_count_metric = Gauge(
|
||||
"block_count", "Number of processed blocks", namespace=NAMESPACE
|
||||
)
|
||||
block_update_time_metric = Histogram(
|
||||
"block_time", "Block update times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
reorg_count_metric = Gauge(
|
||||
"reorg_count", "Number of reorgs", namespace=NAMESPACE
|
||||
)
|
||||
|
||||
def __init__(self, env, db, daemon, notifications):
|
||||
self.env = env
|
||||
self.db = db
|
||||
@@ -162,7 +144,6 @@ class BlockProcessor:
|
||||
self.blocks_event = asyncio.Event()
|
||||
self.prefetcher = Prefetcher(daemon, env.coin, self.blocks_event)
|
||||
self.logger = class_logger(__name__, self.__class__.__name__)
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
|
||||
# Meta
|
||||
self.next_cache_check = 0
|
||||
@@ -171,8 +152,7 @@ class BlockProcessor:
|
||||
|
||||
# Caches of unflushed items.
|
||||
self.headers = []
|
||||
self.block_hashes = []
|
||||
self.block_txs = []
|
||||
self.tx_hashes = []
|
||||
self.undo_infos = []
|
||||
|
||||
# UTXO cache
|
||||
@@ -183,10 +163,6 @@ class BlockProcessor:
|
||||
# is consistent with self.height
|
||||
self.state_lock = asyncio.Lock()
|
||||
|
||||
self.search_cache = {}
|
||||
self.history_cache = {}
|
||||
self.status_server = StatusServer()
|
||||
|
||||
async def run_in_thread_with_lock(self, func, *args):
|
||||
# Run in a thread to prevent blocking. Shielded so that
|
||||
# cancellations from shutdown don't lose work - when the task
|
||||
@@ -195,7 +171,7 @@ class BlockProcessor:
|
||||
# consistent and not being updated elsewhere.
|
||||
async def run_in_thread_locked():
|
||||
async with self.state_lock:
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, func, *args)
|
||||
return await asyncio.get_event_loop().run_in_executor(None, func, *args)
|
||||
return await asyncio.shield(run_in_thread_locked())
|
||||
|
||||
async def check_and_advance_blocks(self, raw_blocks):
|
||||
@@ -212,26 +188,15 @@ class BlockProcessor:
|
||||
chain = [self.tip] + [self.coin.header_hash(h) for h in headers[:-1]]
|
||||
|
||||
if hprevs == chain:
|
||||
start = time.perf_counter()
|
||||
start = time.time()
|
||||
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
|
||||
if self.sql:
|
||||
await self.db.search_index.claim_consumer(self.sql.claim_producer())
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
self.history_cache.clear()
|
||||
self.notifications.notified_mempool_txs.clear()
|
||||
await self._maybe_flush()
|
||||
processed_time = time.perf_counter() - start
|
||||
self.block_count_metric.set(self.height)
|
||||
self.block_update_time_metric.observe(processed_time)
|
||||
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
|
||||
if not self.db.first_sync:
|
||||
s = '' if len(blocks) == 1 else 's'
|
||||
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
|
||||
self.logger.info('processed {:,d} block{} in {:.1f}s'
|
||||
.format(len(blocks), s,
|
||||
time.time() - start))
|
||||
if self._caught_up_event.is_set():
|
||||
if self.sql:
|
||||
await self.db.search_index.apply_filters(self.sql.blocked_streams, self.sql.blocked_channels,
|
||||
self.sql.filtered_streams, self.sql.filtered_channels)
|
||||
await self.notifications.on_block(self.touched, self.height)
|
||||
self.touched = set()
|
||||
elif hprevs[0] != chain[0]:
|
||||
@@ -246,7 +211,7 @@ class BlockProcessor:
|
||||
'resetting the prefetcher')
|
||||
await self.prefetcher.reset_height(self.height)
|
||||
|
||||
async def reorg_chain(self, count: Optional[int] = None):
|
||||
async def reorg_chain(self, count=None):
|
||||
"""Handle a chain reorganisation.
|
||||
|
||||
Count is the number of blocks to simulate a reorg, or None for
|
||||
@@ -255,12 +220,12 @@ class BlockProcessor:
|
||||
self.logger.info('chain reorg detected')
|
||||
else:
|
||||
self.logger.info(f'faking a reorg of {count:,d} blocks')
|
||||
|
||||
await self.flush(True)
|
||||
|
||||
async def get_raw_blocks(last_height, hex_hashes):
|
||||
heights = range(last_height, last_height - len(hex_hashes), -1)
|
||||
try:
|
||||
blocks = [await self.db.read_raw_block(height) for height in heights]
|
||||
blocks = [self.db.read_raw_block(height) for height in heights]
|
||||
self.logger.info(f'read {len(blocks)} blocks from disk')
|
||||
return blocks
|
||||
except FileNotFoundError:
|
||||
@@ -272,27 +237,15 @@ class BlockProcessor:
|
||||
self.touched.discard(None)
|
||||
self.db.flush_backup(self.flush_data(), self.touched)
|
||||
|
||||
try:
|
||||
await self.flush(True)
|
||||
|
||||
start, last, hashes = await self.reorg_hashes(count)
|
||||
# Reverse and convert to hex strings.
|
||||
hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)]
|
||||
self.logger.info("reorg %i block hashes", len(hashes))
|
||||
for hex_hashes in chunks(hashes, 50):
|
||||
raw_blocks = await get_raw_blocks(last, hex_hashes)
|
||||
self.logger.info("got %i raw blocks", len(raw_blocks))
|
||||
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
|
||||
await self.run_in_thread_with_lock(flush_backup)
|
||||
last -= len(raw_blocks)
|
||||
|
||||
await self.prefetcher.reset_height(self.height)
|
||||
self.reorg_count_metric.inc()
|
||||
except:
|
||||
self.logger.exception("boom")
|
||||
raise
|
||||
finally:
|
||||
self.logger.info("done with reorg")
|
||||
start, last, hashes = await self.reorg_hashes(count)
|
||||
# Reverse and convert to hex strings.
|
||||
hashes = [hash_to_hex_str(hash) for hash in reversed(hashes)]
|
||||
for hex_hashes in chunks(hashes, 50):
|
||||
raw_blocks = await get_raw_blocks(last, hex_hashes)
|
||||
await self.run_in_thread_with_lock(self.backup_blocks, raw_blocks)
|
||||
await self.run_in_thread_with_lock(flush_backup)
|
||||
last -= len(raw_blocks)
|
||||
await self.prefetcher.reset_height(self.height)
|
||||
|
||||
async def reorg_hashes(self, count):
|
||||
"""Return a pair (start, last, hashes) of blocks to back up during a
|
||||
@@ -309,7 +262,7 @@ class BlockProcessor:
|
||||
|
||||
return start, last, await self.db.fs_block_hashes(start, count)
|
||||
|
||||
async def calc_reorg_range(self, count: Optional[int]):
|
||||
async def calc_reorg_range(self, count):
|
||||
"""Calculate the reorg range"""
|
||||
|
||||
def diff_pos(hashes1, hashes2):
|
||||
@@ -355,8 +308,8 @@ class BlockProcessor:
|
||||
def flush_data(self):
|
||||
"""The data for a flush. The lock must be taken."""
|
||||
assert self.state_lock.locked()
|
||||
return FlushData(self.height, self.tx_count, self.headers, self.block_hashes,
|
||||
self.block_txs, self.undo_infos, self.utxo_cache,
|
||||
return FlushData(self.height, self.tx_count, self.headers,
|
||||
self.tx_hashes, self.undo_infos, self.utxo_cache,
|
||||
self.db_deletes, self.tip)
|
||||
|
||||
async def flush(self, flush_utxos):
|
||||
@@ -370,9 +323,11 @@ class BlockProcessor:
|
||||
# performed on the DB.
|
||||
if self._caught_up_event.is_set():
|
||||
await self.flush(True)
|
||||
elif time.perf_counter() > self.next_cache_check:
|
||||
await self.flush(True)
|
||||
self.next_cache_check = time.perf_counter() + 30
|
||||
elif time.time() > self.next_cache_check:
|
||||
flush_arg = self.check_cache_size()
|
||||
if flush_arg is not None:
|
||||
await self.flush(flush_arg)
|
||||
self.next_cache_check = time.time() + 30
|
||||
|
||||
def check_cache_size(self):
|
||||
"""Flush a cache if it gets too big."""
|
||||
@@ -411,8 +366,7 @@ class BlockProcessor:
|
||||
for block in blocks:
|
||||
height += 1
|
||||
undo_info = self.advance_txs(
|
||||
height, block.transactions, self.coin.electrum_header(block.header, height),
|
||||
self.coin.header_hash(block.header)
|
||||
height, block.transactions, self.coin.electrum_header(block.header, height)
|
||||
)
|
||||
if height >= min_height:
|
||||
self.undo_infos.append((undo_info, height))
|
||||
@@ -423,26 +377,25 @@ class BlockProcessor:
|
||||
self.headers.extend(headers)
|
||||
self.tip = self.coin.header_hash(headers[-1])
|
||||
|
||||
def advance_txs(self, height, txs, header, block_hash):
|
||||
self.block_hashes.append(block_hash)
|
||||
self.block_txs.append((b''.join(tx_hash for tx, tx_hash in txs), [tx.raw for tx, _ in txs]))
|
||||
|
||||
undo_info = []
|
||||
tx_num = self.tx_count
|
||||
hashXs_by_tx = []
|
||||
def advance_txs(self, height, txs, header):
|
||||
self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs))
|
||||
|
||||
# Use local vars for speed in the loops
|
||||
undo_info = []
|
||||
tx_num = self.tx_count
|
||||
script_hashX = self.coin.hashX_from_script
|
||||
s_pack = pack
|
||||
put_utxo = self.utxo_cache.__setitem__
|
||||
spend_utxo = self.spend_utxo
|
||||
undo_info_append = undo_info.append
|
||||
update_touched = self.touched.update
|
||||
append_hashX_by_tx = hashXs_by_tx.append
|
||||
hashX_from_script = self.coin.hashX_from_script
|
||||
hashXs_by_tx = []
|
||||
append_hashXs = hashXs_by_tx.append
|
||||
|
||||
for tx, tx_hash in txs:
|
||||
hashXs = []
|
||||
append_hashX = hashXs.append
|
||||
tx_numb = pack('<I', tx_num)
|
||||
tx_numb = s_pack('<I', tx_num)
|
||||
|
||||
# Spend the inputs
|
||||
for txin in tx.inputs:
|
||||
@@ -455,17 +408,18 @@ class BlockProcessor:
|
||||
# Add the new UTXOs
|
||||
for idx, txout in enumerate(tx.outputs):
|
||||
# Get the hashX. Ignore unspendable outputs
|
||||
hashX = hashX_from_script(txout.pk_script)
|
||||
hashX = script_hashX(txout.pk_script)
|
||||
if hashX:
|
||||
append_hashX(hashX)
|
||||
put_utxo(tx_hash + pack('<H', idx), hashX + tx_numb + pack('<Q', txout.value))
|
||||
put_utxo(tx_hash + s_pack('<H', idx),
|
||||
hashX + tx_numb + s_pack('<Q', txout.value))
|
||||
|
||||
append_hashX_by_tx(hashXs)
|
||||
append_hashXs(hashXs)
|
||||
update_touched(hashXs)
|
||||
self.db.total_transactions.append(tx_hash)
|
||||
tx_num += 1
|
||||
|
||||
self.db.history.add_unflushed(hashXs_by_tx, self.tx_count)
|
||||
|
||||
self.tx_count = tx_num
|
||||
self.db.tx_counts.append(tx_num)
|
||||
|
||||
@@ -482,7 +436,6 @@ class BlockProcessor:
|
||||
|
||||
coin = self.coin
|
||||
for raw_block in raw_blocks:
|
||||
self.logger.info("backup block %i", self.height)
|
||||
# Check and update self.tip
|
||||
block = coin.block(raw_block, self.height)
|
||||
header_hash = coin.header_hash(block.header)
|
||||
@@ -508,16 +461,20 @@ class BlockProcessor:
|
||||
|
||||
# Use local vars for speed in the loops
|
||||
s_pack = pack
|
||||
put_utxo = self.utxo_cache.__setitem__
|
||||
spend_utxo = self.spend_utxo
|
||||
script_hashX = self.coin.hashX_from_script
|
||||
touched = self.touched
|
||||
undo_entry_len = 12 + HASHX_LEN
|
||||
|
||||
for tx, tx_hash in reversed(txs):
|
||||
for idx, txout in enumerate(tx.outputs):
|
||||
# Spend the TX outputs. Be careful with unspendable
|
||||
# outputs - we didn't save those in the first place.
|
||||
hashX = self.coin.hashX_from_script(txout.pk_script)
|
||||
hashX = script_hashX(txout.pk_script)
|
||||
if hashX:
|
||||
cache_value = self.spend_utxo(tx_hash, idx)
|
||||
self.touched.add(cache_value[:-12])
|
||||
cache_value = spend_utxo(tx_hash, idx)
|
||||
touched.add(cache_value[:-12])
|
||||
|
||||
# Restore the inputs
|
||||
for txin in reversed(tx.inputs):
|
||||
@@ -525,10 +482,9 @@ class BlockProcessor:
|
||||
continue
|
||||
n -= undo_entry_len
|
||||
undo_item = undo_info[n:n + undo_entry_len]
|
||||
self.utxo_cache[txin.prev_hash + s_pack('<H', txin.prev_idx)] = undo_item
|
||||
self.touched.add(undo_item[:-12])
|
||||
|
||||
self.db.total_transactions.pop()
|
||||
put_utxo(txin.prev_hash + s_pack('<H', txin.prev_idx),
|
||||
undo_item)
|
||||
touched.add(undo_item[:-12])
|
||||
|
||||
assert n == 0
|
||||
self.tx_count -= len(txs)
|
||||
@@ -605,18 +561,15 @@ class BlockProcessor:
|
||||
# Key: b'h' + compressed_tx_hash + tx_idx + tx_num
|
||||
# Value: hashX
|
||||
prefix = b'h' + tx_hash[:4] + idx_packed
|
||||
candidates = dict(self.db.utxo_db.iterator(prefix=prefix))
|
||||
candidates = {db_key: hashX for db_key, hashX
|
||||
in self.db.utxo_db.iterator(prefix=prefix)}
|
||||
|
||||
for hdb_key, hashX in candidates.items():
|
||||
tx_num_packed = hdb_key[-4:]
|
||||
if len(candidates) > 1:
|
||||
|
||||
if len(candidates) > 1:
|
||||
tx_num, = unpack('<I', tx_num_packed)
|
||||
try:
|
||||
hash, height = self.db.fs_tx_hash(tx_num)
|
||||
except IndexError:
|
||||
self.logger.error("data integrity error for hashx history: %s missing tx #%s (%s:%s)",
|
||||
hashX.hex(), tx_num, hash_to_hex_str(tx_hash), tx_idx)
|
||||
continue
|
||||
hash, height = self.db.fs_tx_hash(tx_num)
|
||||
if hash != tx_hash:
|
||||
assert hash is not None # Should always be found
|
||||
continue
|
||||
@@ -625,17 +578,12 @@ class BlockProcessor:
|
||||
# Value: the UTXO value as a 64-bit unsigned integer
|
||||
udb_key = b'u' + hashX + hdb_key[-6:]
|
||||
utxo_value_packed = self.db.utxo_db.get(udb_key)
|
||||
if utxo_value_packed is None:
|
||||
self.logger.warning(
|
||||
"%s:%s is not found in UTXO db for %s", hash_to_hex_str(tx_hash), tx_idx, hash_to_hex_str(hashX)
|
||||
)
|
||||
raise ChainError(f"{hash_to_hex_str(tx_hash)}:{tx_idx} is not found in UTXO db for {hash_to_hex_str(hashX)}")
|
||||
# Remove both entries for this UTXO
|
||||
self.db_deletes.append(hdb_key)
|
||||
self.db_deletes.append(udb_key)
|
||||
return hashX + tx_num_packed + utxo_value_packed
|
||||
if utxo_value_packed:
|
||||
# Remove both entries for this UTXO
|
||||
self.db_deletes.append(hdb_key)
|
||||
self.db_deletes.append(udb_key)
|
||||
return hashX + tx_num_packed + utxo_value_packed
|
||||
|
||||
self.logger.error('UTXO {hash_to_hex_str(tx_hash)} / {tx_idx} not found in "h" table')
|
||||
raise ChainError('UTXO {} / {:,d} not found in "h" table'
|
||||
.format(hash_to_hex_str(tx_hash), tx_idx))
|
||||
|
||||
@@ -648,16 +596,12 @@ class BlockProcessor:
|
||||
self._caught_up_event.set()
|
||||
await self.blocks_event.wait()
|
||||
self.blocks_event.clear()
|
||||
if self.reorg_count: # this could only happen by calling the reorg rpc
|
||||
if self.reorg_count:
|
||||
await self.reorg_chain(self.reorg_count)
|
||||
self.reorg_count = 0
|
||||
else:
|
||||
blocks = self.prefetcher.get_prefetched_blocks()
|
||||
try:
|
||||
await self.check_and_advance_blocks(blocks)
|
||||
except Exception:
|
||||
self.logger.exception("error while processing txs")
|
||||
raise
|
||||
await self.check_and_advance_blocks(blocks)
|
||||
|
||||
async def _first_caught_up(self):
|
||||
self.logger.info(f'caught up to height {self.height}')
|
||||
@@ -691,11 +635,9 @@ class BlockProcessor:
|
||||
disk before exiting, as otherwise a significant amount of work
|
||||
could be lost.
|
||||
"""
|
||||
|
||||
self._caught_up_event = caught_up_event
|
||||
try:
|
||||
await self._first_open_dbs()
|
||||
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
|
||||
await asyncio.wait([
|
||||
self.prefetcher.main_loop(self.height),
|
||||
self._process_prefetched_blocks()
|
||||
@@ -706,12 +648,10 @@ class BlockProcessor:
|
||||
self.logger.exception("Block processing failed!")
|
||||
raise
|
||||
finally:
|
||||
self.status_server.stop()
|
||||
# Shut down block processing
|
||||
self.logger.info('flushing to DB for a clean shutdown...')
|
||||
await self.flush(True)
|
||||
self.db.close()
|
||||
self.executor.shutdown(wait=True)
|
||||
|
||||
def force_chain_reorg(self, count):
|
||||
"""Force a reorg of the given number of blocks.
|
||||
@@ -726,6 +666,7 @@ class BlockProcessor:
|
||||
|
||||
|
||||
class Timer:
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.total = 0
|
||||
@@ -786,29 +727,43 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
|
||||
self.sql: SQLDB = self.db.sql
|
||||
self.timer = Timer('BlockProcessor')
|
||||
self.search_cache = {}
|
||||
|
||||
def advance_blocks(self, blocks):
|
||||
if self.sql:
|
||||
self.sql.begin()
|
||||
self.sql.begin()
|
||||
try:
|
||||
self.timer.run(super().advance_blocks, blocks)
|
||||
except:
|
||||
self.logger.exception(f'Error while advancing transaction in new block.')
|
||||
raise
|
||||
finally:
|
||||
if self.sql:
|
||||
self.sql.commit()
|
||||
if self.sql and self.db.first_sync and self.height == self.daemon.cached_height():
|
||||
self.sql.commit()
|
||||
if self.db.first_sync and self.height == self.daemon.cached_height():
|
||||
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
|
||||
if self.env.individual_tag_indexes:
|
||||
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
||||
self.timer.run(self.sql.execute, self.sql.LANGUAGE_INDEXES, timer_name='executing LANGUAGE_INDEXES')
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
|
||||
def advance_txs(self, height, txs, header, block_hash):
|
||||
def advance_txs(self, height, txs, header):
|
||||
timer = self.timer.sub_timers['advance_blocks']
|
||||
undo = timer.run(super().advance_txs, height, txs, header, block_hash, timer_name='super().advance_txs')
|
||||
if self.sql:
|
||||
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
|
||||
undo = timer.run(super().advance_txs, height, txs, header, timer_name='super().advance_txs')
|
||||
timer.run(self.sql.advance_txs, height, txs, header, self.daemon.cached_height(), forward_timer=True)
|
||||
if (height % 10000 == 0 or not self.db.first_sync) and self.logger.isEnabledFor(10):
|
||||
self.timer.show(height=height)
|
||||
return undo
|
||||
|
||||
def _checksig(self, value, address):
|
||||
try:
|
||||
claim_dict = Claim.from_bytes(value)
|
||||
cert_id = claim_dict.signing_channel_hash
|
||||
if not self.should_validate_signatures:
|
||||
return cert_id
|
||||
if cert_id:
|
||||
cert_claim = self.db.get_claim_info(cert_id)
|
||||
if cert_claim:
|
||||
certificate = Claim.from_bytes(cert_claim.value)
|
||||
claim_dict.validate_signature(address, certificate)
|
||||
return cert_id
|
||||
except Exception:
|
||||
pass
|
||||
@@ -8,7 +8,7 @@ from lbry.wallet.server.server import Server
|
||||
|
||||
def get_argument_parser():
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="lbry-hub"
|
||||
prog="torba-server"
|
||||
)
|
||||
parser.add_argument("spvserver", type=str, help="Python class path to SPV server implementation.",
|
||||
nargs="?", default="lbry.wallet.server.coin.LBC")
|
||||
@@ -25,10 +25,8 @@ def main():
|
||||
parser = get_argument_parser()
|
||||
args = parser.parse_args()
|
||||
coin_class = get_coin_class(args.spvserver)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-4s %(name)s:%(lineno)d: %(message)s")
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logging.info('lbry.server starting')
|
||||
logging.getLogger('aiohttp').setLevel(logging.WARNING)
|
||||
logging.getLogger('elasticsearch').setLevel(logging.WARNING)
|
||||
try:
|
||||
server = Server(Env(coin_class))
|
||||
server.run()
|
||||
|
||||
@@ -5,8 +5,7 @@ import time
|
||||
from functools import wraps
|
||||
|
||||
import aiohttp
|
||||
from prometheus_client import Gauge, Histogram
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
|
||||
from lbry.wallet.rpc.jsonrpc import RPCError
|
||||
from lbry.wallet.server.util import hex_to_bytes, class_logger
|
||||
from lbry.wallet.rpc import JSONRPC
|
||||
@@ -24,23 +23,12 @@ class WorkQueueFullError(Exception):
|
||||
"""Internal - when the daemon's work queue is full."""
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
|
||||
|
||||
class Daemon:
|
||||
"""Handles connections to a daemon at the given URL."""
|
||||
|
||||
WARMING_UP = -28
|
||||
id_counter = itertools.count()
|
||||
|
||||
lbrycrd_request_time_metric = Histogram(
|
||||
"lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",)
|
||||
)
|
||||
lbrycrd_pending_count_metric = Gauge(
|
||||
"lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE,
|
||||
labelnames=("method",)
|
||||
)
|
||||
|
||||
def __init__(self, coin, url, max_workqueue=10, init_retry=0.25,
|
||||
max_retry=4.0):
|
||||
self.coin = coin
|
||||
@@ -54,8 +42,6 @@ class Daemon:
|
||||
self._height = None
|
||||
self.available_rpcs = {}
|
||||
self.connector = aiohttp.TCPConnector()
|
||||
self._block_hash_cache = LRUCacheWithMetrics(100000)
|
||||
self._block_cache = LRUCacheWithMetrics(2 ** 16, metric_name='block', namespace=NAMESPACE)
|
||||
|
||||
async def close(self):
|
||||
if self.connector:
|
||||
@@ -120,7 +106,6 @@ class Daemon:
|
||||
Handles temporary connection issues. Daemon response errors
|
||||
are raise through DaemonError.
|
||||
"""
|
||||
|
||||
def log_error(error):
|
||||
nonlocal last_error_log, retry
|
||||
now = time.time()
|
||||
@@ -134,13 +119,8 @@ class Daemon:
|
||||
last_error_log = 0
|
||||
data = json.dumps(payload)
|
||||
retry = self.init_retry
|
||||
methods = tuple(
|
||||
[payload['method']] if isinstance(payload, dict) else [request['method'] for request in payload]
|
||||
)
|
||||
while True:
|
||||
try:
|
||||
for method in methods:
|
||||
self.lbrycrd_pending_count_metric.labels(method=method).inc()
|
||||
result = await self._send_data(data)
|
||||
result = processor(result)
|
||||
if on_good_message:
|
||||
@@ -163,17 +143,12 @@ class Daemon:
|
||||
except WorkQueueFullError:
|
||||
log_error('work queue full.')
|
||||
on_good_message = 'running normally'
|
||||
finally:
|
||||
for method in methods:
|
||||
self.lbrycrd_pending_count_metric.labels(method=method).dec()
|
||||
|
||||
await asyncio.sleep(retry)
|
||||
retry = max(min(self.max_retry, retry * 2), self.init_retry)
|
||||
|
||||
async def _send_single(self, method, params=None):
|
||||
"""Send a single request to the daemon."""
|
||||
|
||||
start = time.perf_counter()
|
||||
|
||||
def processor(result):
|
||||
err = result['error']
|
||||
if not err:
|
||||
@@ -185,9 +160,7 @@ class Daemon:
|
||||
payload = {'method': method, 'id': next(self.id_counter)}
|
||||
if params:
|
||||
payload['params'] = params
|
||||
result = await self._send(payload, processor)
|
||||
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
|
||||
return result
|
||||
return await self._send(payload, processor)
|
||||
|
||||
async def _send_vector(self, method, params_iterable, replace_errs=False):
|
||||
"""Send several requests of the same method.
|
||||
@@ -195,9 +168,6 @@ class Daemon:
|
||||
The result will be an array of the same length as params_iterable.
|
||||
If replace_errs is true, any item with an error is returned as None,
|
||||
otherwise an exception is raised."""
|
||||
|
||||
start = time.perf_counter()
|
||||
|
||||
def processor(result):
|
||||
errs = [item['error'] for item in result if item['error']]
|
||||
if any(err.get('code') == self.WARMING_UP for err in errs):
|
||||
@@ -208,11 +178,9 @@ class Daemon:
|
||||
|
||||
payload = [{'method': method, 'params': p, 'id': next(self.id_counter)}
|
||||
for p in params_iterable]
|
||||
result = []
|
||||
if payload:
|
||||
result = await self._send(payload, processor)
|
||||
self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start)
|
||||
return result
|
||||
return await self._send(payload, processor)
|
||||
return []
|
||||
|
||||
async def _is_rpc_available(self, method):
|
||||
"""Return whether given RPC method is available in the daemon.
|
||||
@@ -233,27 +201,12 @@ class Daemon:
|
||||
|
||||
async def block_hex_hashes(self, first, count):
|
||||
"""Return the hex hashes of count block starting at height first."""
|
||||
if first + count < (self.cached_height() or 0) - 200:
|
||||
return await self._cached_block_hex_hashes(first, count)
|
||||
params_iterable = ((h, ) for h in range(first, first + count))
|
||||
return await self._send_vector('getblockhash', params_iterable)
|
||||
|
||||
async def _cached_block_hex_hashes(self, first, count):
|
||||
"""Return the hex hashes of count block starting at height first."""
|
||||
cached = self._block_hash_cache.get((first, count))
|
||||
if cached:
|
||||
return cached
|
||||
params_iterable = ((h, ) for h in range(first, first + count))
|
||||
self._block_hash_cache[(first, count)] = await self._send_vector('getblockhash', params_iterable)
|
||||
return self._block_hash_cache[(first, count)]
|
||||
|
||||
async def deserialised_block(self, hex_hash):
|
||||
"""Return the deserialised block with the given hex hash."""
|
||||
if hex_hash not in self._block_cache:
|
||||
block = await self._send_single('getblock', (hex_hash, True))
|
||||
self._block_cache[hex_hash] = block
|
||||
return block
|
||||
return self._block_cache[hex_hash]
|
||||
return await self._send_single('getblock', (hex_hash, True))
|
||||
|
||||
async def raw_blocks(self, hex_hashes):
|
||||
"""Return the raw binary blocks with the given hex hashes."""
|
||||
|
||||
+201
-401
@@ -1,8 +1,6 @@
|
||||
CLAIM_TYPES = {
|
||||
'stream': 1,
|
||||
'channel': 2,
|
||||
'repost': 3,
|
||||
'collection': 4,
|
||||
}
|
||||
|
||||
STREAM_TYPES = {
|
||||
@@ -11,410 +9,212 @@ STREAM_TYPES = {
|
||||
'image': 3,
|
||||
'document': 4,
|
||||
'binary': 5,
|
||||
'model': 6,
|
||||
}
|
||||
|
||||
# 9/21/2020
|
||||
MOST_USED_TAGS = {
|
||||
"gaming",
|
||||
"people & blogs",
|
||||
"entertainment",
|
||||
"music",
|
||||
"pop culture",
|
||||
"education",
|
||||
"technology",
|
||||
"blockchain",
|
||||
"news",
|
||||
"funny",
|
||||
"science & technology",
|
||||
"learning",
|
||||
"gameplay",
|
||||
"news & politics",
|
||||
"comedy",
|
||||
"bitcoin",
|
||||
"beliefs",
|
||||
"nature",
|
||||
"art",
|
||||
"economics",
|
||||
"film & animation",
|
||||
"lets play",
|
||||
"games",
|
||||
"sports",
|
||||
"howto & style",
|
||||
"game",
|
||||
"cryptocurrency",
|
||||
"playstation 4",
|
||||
"automotive",
|
||||
"crypto",
|
||||
"mature",
|
||||
"sony interactive entertainment",
|
||||
"walkthrough",
|
||||
"tutorial",
|
||||
"video game",
|
||||
"weapons",
|
||||
"playthrough",
|
||||
"pc",
|
||||
"anime",
|
||||
"how to",
|
||||
"btc",
|
||||
"fun",
|
||||
"ethereum",
|
||||
"food",
|
||||
"travel & events",
|
||||
"minecraft",
|
||||
"science",
|
||||
"autos & vehicles",
|
||||
"play",
|
||||
"politics",
|
||||
"commentary",
|
||||
"twitch",
|
||||
"ps4live",
|
||||
"love",
|
||||
"ps4",
|
||||
"nonprofits & activism",
|
||||
"ps4share",
|
||||
"fortnite",
|
||||
"xbox",
|
||||
"porn",
|
||||
"video games",
|
||||
"trump",
|
||||
"español",
|
||||
"money",
|
||||
"music video",
|
||||
"nintendo",
|
||||
"movie",
|
||||
"coronavirus",
|
||||
"donald trump",
|
||||
"steam",
|
||||
"trailer",
|
||||
"android",
|
||||
"podcast",
|
||||
"xbox one",
|
||||
"survival",
|
||||
"audio",
|
||||
"linux",
|
||||
"travel",
|
||||
"funny moments",
|
||||
"litecoin",
|
||||
"animation",
|
||||
"gamer",
|
||||
"lets",
|
||||
"playstation",
|
||||
"bitcoin news",
|
||||
"history",
|
||||
"xxx",
|
||||
"fox news",
|
||||
"dance",
|
||||
"god",
|
||||
"adventure",
|
||||
"liberal",
|
||||
"2020",
|
||||
"horror",
|
||||
"government",
|
||||
"freedom",
|
||||
"reaction",
|
||||
"meme",
|
||||
"photography",
|
||||
"truth",
|
||||
"health",
|
||||
"lbry",
|
||||
"family",
|
||||
"online",
|
||||
"eth",
|
||||
"crypto news",
|
||||
"diy",
|
||||
"trading",
|
||||
"gold",
|
||||
"memes",
|
||||
"world",
|
||||
"space",
|
||||
"lol",
|
||||
"covid-19",
|
||||
"rpg",
|
||||
"humor",
|
||||
"democrat",
|
||||
"film",
|
||||
"call of duty",
|
||||
"tech",
|
||||
"religion",
|
||||
"conspiracy",
|
||||
"rap",
|
||||
"cnn",
|
||||
"hangoutsonair",
|
||||
"unboxing",
|
||||
"fiction",
|
||||
"conservative",
|
||||
"cars",
|
||||
"hoa",
|
||||
"epic",
|
||||
"programming",
|
||||
"progressive",
|
||||
"cryptocurrency news",
|
||||
"classical",
|
||||
"jesus",
|
||||
"movies",
|
||||
"book",
|
||||
"ps3",
|
||||
"republican",
|
||||
"fitness",
|
||||
"books",
|
||||
"multiplayer",
|
||||
"animals",
|
||||
"pokemon",
|
||||
"bitcoin price",
|
||||
"facebook",
|
||||
"sharefactory",
|
||||
"criptomonedas",
|
||||
"cod",
|
||||
"bible",
|
||||
"business",
|
||||
"stream",
|
||||
"comics",
|
||||
"how",
|
||||
"fail",
|
||||
"nsfw",
|
||||
"new music",
|
||||
"satire",
|
||||
"pets & animals",
|
||||
"computer",
|
||||
"classical music",
|
||||
"indie",
|
||||
"musica",
|
||||
"msnbc",
|
||||
"fps",
|
||||
"mod",
|
||||
"sport",
|
||||
"sony",
|
||||
"ripple",
|
||||
"auto",
|
||||
"rock",
|
||||
"marvel",
|
||||
"complete",
|
||||
"mining",
|
||||
"political",
|
||||
"mobile",
|
||||
"pubg",
|
||||
"hip hop",
|
||||
"flat earth",
|
||||
"xbox 360",
|
||||
"reviews",
|
||||
"vlogging",
|
||||
"latest news",
|
||||
"hack",
|
||||
"tarot",
|
||||
"iphone",
|
||||
"media",
|
||||
"cute",
|
||||
"christian",
|
||||
"free speech",
|
||||
"trap",
|
||||
"war",
|
||||
"remix",
|
||||
"ios",
|
||||
"xrp",
|
||||
"spirituality",
|
||||
"song",
|
||||
"league of legends",
|
||||
"cat"
|
||||
'model': 6
|
||||
}
|
||||
|
||||
MATURE_TAGS = [
|
||||
'nsfw', 'porn', 'xxx', 'mature', 'adult', 'sex'
|
||||
]
|
||||
|
||||
|
||||
def normalize_tag(tag):
|
||||
return tag.replace(" ", "_").replace("&", "and").replace("-", "_")
|
||||
|
||||
|
||||
COMMON_TAGS = {
|
||||
tag: normalize_tag(tag) for tag in list(MOST_USED_TAGS)
|
||||
"gaming": "gaming",
|
||||
"people & blogs": "people_and_blogs",
|
||||
"pop culture": "pop_culture",
|
||||
"entertainment": "entertainment",
|
||||
"technology": "technology",
|
||||
"music": "music",
|
||||
"funny": "funny",
|
||||
"education": "education",
|
||||
"learning": "learning",
|
||||
"news": "news",
|
||||
"gameplay": "gameplay",
|
||||
"science & technology": "science_and_technology",
|
||||
"playstation 4": "playstation_4",
|
||||
"beliefs": "beliefs",
|
||||
"nature": "nature",
|
||||
"news & politics": "news_and_politics",
|
||||
"comedy": "comedy",
|
||||
"games": "games",
|
||||
"sony interactive entertainment": "sony_interactive_entertainment",
|
||||
"film & animation": "film_and_animation",
|
||||
"game": "game",
|
||||
"howto & style": "howto_and_style",
|
||||
"weapons": "weapons",
|
||||
"blockchain": "blockchain",
|
||||
"video game": "video_game",
|
||||
"sports": "sports",
|
||||
"walkthrough": "walkthrough",
|
||||
"ps4live": "ps4live",
|
||||
"art": "art",
|
||||
"pc": "pc",
|
||||
"economics": "economics",
|
||||
"automotive": "automotive",
|
||||
"minecraft": "minecraft",
|
||||
"playthrough": "playthrough",
|
||||
"ps4share": "ps4share",
|
||||
"tutorial": "tutorial",
|
||||
"play": "play",
|
||||
"twitch": "twitch",
|
||||
"how to": "how_to",
|
||||
"ps4": "ps4",
|
||||
"bitcoin": "bitcoin",
|
||||
"fortnite": "fortnite",
|
||||
"commentary": "commentary",
|
||||
"lets play": "lets_play",
|
||||
"fun": "fun",
|
||||
"politics": "politics",
|
||||
"xbox": "xbox",
|
||||
"autos & vehicles": "autos_and_vehicles",
|
||||
"travel & events": "travel_and_events",
|
||||
"food": "food",
|
||||
"science": "science",
|
||||
"mature": "mature",
|
||||
"xbox one": "xbox_one",
|
||||
"liberal": "liberal",
|
||||
"democrat": "democrat",
|
||||
"progressive": "progressive",
|
||||
"survival": "survival",
|
||||
"nonprofits & activism": "nonprofits_and_activism",
|
||||
"cryptocurrency": "cryptocurrency",
|
||||
"playstation": "playstation",
|
||||
"nintendo": "nintendo",
|
||||
"government": "government",
|
||||
"steam": "steam",
|
||||
"podcast": "podcast",
|
||||
"horror": "horror",
|
||||
"conservative": "conservative",
|
||||
"reaction": "reaction",
|
||||
"trailer": "trailer",
|
||||
"love": "love",
|
||||
"cnn": "cnn",
|
||||
"republican": "republican",
|
||||
"gamer": "gamer",
|
||||
"political": "political",
|
||||
"hangoutsonair": "hangoutsonair",
|
||||
"hoa": "hoa",
|
||||
"msnbc": "msnbc",
|
||||
"cbs": "cbs",
|
||||
"donald trump": "donald_trump",
|
||||
"fiction": "fiction",
|
||||
"fox news": "fox_news",
|
||||
"anime": "anime",
|
||||
"crypto": "crypto",
|
||||
"ethereum": "ethereum",
|
||||
"call of duty": "call_of_duty",
|
||||
"multiplayer": "multiplayer",
|
||||
"android": "android",
|
||||
"epic": "epic",
|
||||
"rpg": "rpg",
|
||||
"adventure": "adventure",
|
||||
"secular talk": "secular_talk",
|
||||
"btc": "btc",
|
||||
"atheist": "atheist",
|
||||
"atheism": "atheism",
|
||||
"ps3": "ps3",
|
||||
"video games": "video_games",
|
||||
"cod": "cod",
|
||||
"agnostic": "agnostic",
|
||||
"movie": "movie",
|
||||
"online": "online",
|
||||
"fps": "fps",
|
||||
"mod": "mod",
|
||||
"reviews": "reviews",
|
||||
"sharefactory": "sharefactory",
|
||||
"world": "world",
|
||||
"space": "space",
|
||||
"hilarious": "hilarious",
|
||||
"stream": "stream",
|
||||
"lol": "lol",
|
||||
"sony": "sony",
|
||||
"god": "god",
|
||||
"lets": "lets",
|
||||
"dance": "dance",
|
||||
"pvp": "pvp",
|
||||
"tech": "tech",
|
||||
"zombies": "zombies",
|
||||
"pokemon": "pokemon",
|
||||
"fail": "fail",
|
||||
"xbox 360": "xbox_360",
|
||||
"film": "film",
|
||||
"unboxing": "unboxing",
|
||||
"animation": "animation",
|
||||
"travel": "travel",
|
||||
"money": "money",
|
||||
"wwe": "wwe",
|
||||
"how": "how",
|
||||
"mods": "mods",
|
||||
"pubg": "pubg",
|
||||
"indie": "indie",
|
||||
"strategy": "strategy",
|
||||
"history": "history",
|
||||
"rap": "rap",
|
||||
"ios": "ios",
|
||||
"sony computer entertainment": "sony_computer_entertainment",
|
||||
"mobile": "mobile",
|
||||
"trump": "trump",
|
||||
"flat earth": "flat_earth",
|
||||
"hack": "hack",
|
||||
"trap": "trap",
|
||||
"fox": "fox",
|
||||
"vlogging": "vlogging",
|
||||
"news radio": "news_radio",
|
||||
"humor": "humor",
|
||||
"facebook": "facebook",
|
||||
"edm": "edm",
|
||||
"fitness": "fitness",
|
||||
"vaping": "vaping",
|
||||
"hip hop": "hip_hop",
|
||||
"secular": "secular",
|
||||
"jesus": "jesus",
|
||||
"vape": "vape",
|
||||
"song": "song",
|
||||
"remix": "remix",
|
||||
"guitar": "guitar",
|
||||
"daily": "daily",
|
||||
"mining": "mining",
|
||||
"diy": "diy",
|
||||
"videogame": "videogame",
|
||||
"pets & animals": "pets_and_animals",
|
||||
"funny moments": "funny_moments",
|
||||
"religion": "religion",
|
||||
"death": "death",
|
||||
"media": "media",
|
||||
"nbc": "nbc",
|
||||
"war": "war",
|
||||
"freedom": "freedom",
|
||||
"viral": "viral",
|
||||
"meme": "meme",
|
||||
"family": "family",
|
||||
"gold": "gold",
|
||||
"photography": "photography",
|
||||
"chill": "chill",
|
||||
"zombie": "zombie",
|
||||
"computer": "computer",
|
||||
"sniper": "sniper",
|
||||
"bible": "bible",
|
||||
"linux": "linux",
|
||||
"overwatch": "overwatch",
|
||||
"pro": "pro",
|
||||
"dragon": "dragon",
|
||||
"litecoin": "litecoin",
|
||||
"gta": "gta",
|
||||
"iphone": "iphone",
|
||||
"house": "house",
|
||||
"bass": "bass",
|
||||
"bitcoin news": "bitcoin_news",
|
||||
"wii": "wii",
|
||||
"crash": "crash",
|
||||
"league of legends": "league_of_legends",
|
||||
"grand theft auto v": "grand_theft_auto_v",
|
||||
"mario": "mario",
|
||||
"mmorpg": "mmorpg",
|
||||
"satire": "satire",
|
||||
"fire": "fire",
|
||||
"racing": "racing",
|
||||
"apple": "apple",
|
||||
"health": "health",
|
||||
"instrumental": "instrumental",
|
||||
"destiny": "destiny",
|
||||
"truth": "truth",
|
||||
"race": "race"
|
||||
}
|
||||
|
||||
INDEXED_LANGUAGES = [
|
||||
'none',
|
||||
'en',
|
||||
'aa',
|
||||
'ab',
|
||||
'ae',
|
||||
'af',
|
||||
'ak',
|
||||
'am',
|
||||
'an',
|
||||
'ar',
|
||||
'as',
|
||||
'av',
|
||||
'ay',
|
||||
'az',
|
||||
'ba',
|
||||
'be',
|
||||
'bg',
|
||||
'bh',
|
||||
'bi',
|
||||
'bm',
|
||||
'bn',
|
||||
'bo',
|
||||
'br',
|
||||
'bs',
|
||||
'ca',
|
||||
'ce',
|
||||
'ch',
|
||||
'co',
|
||||
'cr',
|
||||
'cs',
|
||||
'cu',
|
||||
'cv',
|
||||
'cy',
|
||||
'da',
|
||||
'de',
|
||||
'dv',
|
||||
'dz',
|
||||
'ee',
|
||||
'el',
|
||||
'eo',
|
||||
'es',
|
||||
'et',
|
||||
'eu',
|
||||
'fa',
|
||||
'ff',
|
||||
'fi',
|
||||
'fj',
|
||||
'fo',
|
||||
'fr',
|
||||
'fy',
|
||||
'ga',
|
||||
'gd',
|
||||
'gl',
|
||||
'gn',
|
||||
'gu',
|
||||
'gv',
|
||||
'ha',
|
||||
'he',
|
||||
'hi',
|
||||
'ho',
|
||||
'hr',
|
||||
'ht',
|
||||
'hu',
|
||||
'hy',
|
||||
'hz',
|
||||
'ia',
|
||||
'id',
|
||||
'ie',
|
||||
'ig',
|
||||
'ii',
|
||||
'ik',
|
||||
'io',
|
||||
'is',
|
||||
'it',
|
||||
'iu',
|
||||
'ja',
|
||||
'jv',
|
||||
'ka',
|
||||
'kg',
|
||||
'ki',
|
||||
'kj',
|
||||
'kk',
|
||||
'kl',
|
||||
'km',
|
||||
'kn',
|
||||
'ko',
|
||||
'kr',
|
||||
'ks',
|
||||
'ku',
|
||||
'kv',
|
||||
'kw',
|
||||
'ky',
|
||||
'la',
|
||||
'lb',
|
||||
'lg',
|
||||
'li',
|
||||
'ln',
|
||||
'lo',
|
||||
'lt',
|
||||
'lu',
|
||||
'lv',
|
||||
'mg',
|
||||
'mh',
|
||||
'mi',
|
||||
'mk',
|
||||
'ml',
|
||||
'mn',
|
||||
'mr',
|
||||
'ms',
|
||||
'mt',
|
||||
'my',
|
||||
'na',
|
||||
'nb',
|
||||
'nd',
|
||||
'ne',
|
||||
'ng',
|
||||
'nl',
|
||||
'nn',
|
||||
'no',
|
||||
'nr',
|
||||
'nv',
|
||||
'ny',
|
||||
'oc',
|
||||
'oj',
|
||||
'om',
|
||||
'or',
|
||||
'os',
|
||||
'pa',
|
||||
'pi',
|
||||
'pl',
|
||||
'ps',
|
||||
'pt',
|
||||
'qu',
|
||||
'rm',
|
||||
'rn',
|
||||
'ro',
|
||||
'ru',
|
||||
'rw',
|
||||
'sa',
|
||||
'sc',
|
||||
'sd',
|
||||
'se',
|
||||
'sg',
|
||||
'si',
|
||||
'sk',
|
||||
'sl',
|
||||
'sm',
|
||||
'sn',
|
||||
'so',
|
||||
'sq',
|
||||
'sr',
|
||||
'ss',
|
||||
'st',
|
||||
'su',
|
||||
'sv',
|
||||
'sw',
|
||||
'ta',
|
||||
'te',
|
||||
'tg',
|
||||
'th',
|
||||
'ti',
|
||||
'tk',
|
||||
'tl',
|
||||
'tn',
|
||||
'to',
|
||||
'tr',
|
||||
'ts',
|
||||
'tt',
|
||||
'tw',
|
||||
'ty',
|
||||
'ug',
|
||||
'uk',
|
||||
'ur',
|
||||
'uz',
|
||||
've',
|
||||
'vi',
|
||||
'vo',
|
||||
'wa',
|
||||
'wo',
|
||||
'xh',
|
||||
'yi',
|
||||
'yo',
|
||||
'za',
|
||||
'zh',
|
||||
'zu'
|
||||
]
|
||||
@@ -1 +0,0 @@
|
||||
from .search import SearchIndex
|
||||
@@ -1,61 +0,0 @@
|
||||
INDEX_DEFAULT_SETTINGS = {
|
||||
"settings":
|
||||
{"analysis":
|
||||
{"analyzer": {
|
||||
"default": {"tokenizer": "whitespace", "filter": ["lowercase", "porter_stem"]}}},
|
||||
"index":
|
||||
{"refresh_interval": -1,
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0,
|
||||
"sort": {
|
||||
"field": ["trending_mixed", "release_time"],
|
||||
"order": ["desc", "desc"]
|
||||
}}
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"claim_id": {
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"ignore_above": 256,
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"type": "text",
|
||||
"index_prefixes": {
|
||||
"min_chars": 1,
|
||||
"max_chars": 10
|
||||
}
|
||||
},
|
||||
"height": {"type": "integer"},
|
||||
"claim_type": {"type": "byte"},
|
||||
"censor_type": {"type": "byte"},
|
||||
"trending_mixed": {"type": "float"},
|
||||
"release_time": {"type": "long"},
|
||||
}
|
||||
}
|
||||
}
|
||||
FIELDS = {'is_controlling', 'last_take_over_height', 'claim_id', 'claim_name', 'normalized', 'tx_position', 'amount',
|
||||
'timestamp', 'creation_timestamp', 'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'release_time', 'short_url', 'canonical_url', 'title', 'author', 'description', 'claim_type', 'reposted',
|
||||
'stream_type', 'media_type', 'fee_amount', 'fee_currency', 'duration', 'reposted_claim_hash', 'censor_type',
|
||||
'claims_in_channel', 'channel_join', 'signature_valid', 'effective_amount', 'support_amount',
|
||||
'trending_group', 'trending_mixed', 'trending_local', 'trending_global', 'channel_id', 'tx_id', 'tx_nout',
|
||||
'signature', 'signature_digest', 'public_key_bytes', 'public_key_hash', 'public_key_id', '_id', 'tags',
|
||||
'reposted_claim_id'}
|
||||
TEXT_FIELDS = {'author', 'canonical_url', 'channel_id', 'claim_name', 'description', 'claim_id',
|
||||
'media_type', 'normalized', 'public_key_bytes', 'public_key_hash', 'short_url', 'signature',
|
||||
'signature_digest', 'stream_type', 'title', 'tx_id', 'fee_currency', 'reposted_claim_id', 'tags'}
|
||||
RANGE_FIELDS = {
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
|
||||
'tx_position', 'channel_join', 'reposted', 'limit_claims_per_channel',
|
||||
'amount', 'effective_amount', 'support_amount',
|
||||
'trending_group', 'trending_mixed', 'censor_type',
|
||||
'trending_local', 'trending_global',
|
||||
}
|
||||
REPLACEMENTS = {
|
||||
'name': 'normalized',
|
||||
'txid': 'tx_id',
|
||||
'claim_hash': '_id'
|
||||
}
|
||||
@@ -1,641 +0,0 @@
|
||||
import asyncio
|
||||
import struct
|
||||
from binascii import unhexlify
|
||||
from collections import Counter, deque
|
||||
from decimal import Decimal
|
||||
from operator import itemgetter
|
||||
from typing import Optional, List, Iterable, Union
|
||||
|
||||
from elasticsearch import AsyncElasticsearch, NotFoundError, ConnectionError
|
||||
from elasticsearch.helpers import async_streaming_bulk
|
||||
|
||||
from lbry.crypto.base58 import Base58
|
||||
from lbry.error import ResolveCensoredError, claim_id as parse_claim_id
|
||||
from lbry.schema.result import Outputs, Censor
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
from lbry.utils import LRUCache
|
||||
from lbry.wallet.server.db.common import CLAIM_TYPES, STREAM_TYPES
|
||||
from lbry.wallet.server.db.elasticsearch.constants import INDEX_DEFAULT_SETTINGS, REPLACEMENTS, FIELDS, TEXT_FIELDS, \
|
||||
RANGE_FIELDS
|
||||
from lbry.wallet.server.util import class_logger
|
||||
|
||||
|
||||
class ChannelResolution(str):
|
||||
@classmethod
|
||||
def lookup_error(cls, url):
|
||||
return LookupError(f'Could not find channel in "{url}".')
|
||||
|
||||
|
||||
class StreamResolution(str):
|
||||
@classmethod
|
||||
def lookup_error(cls, url):
|
||||
return LookupError(f'Could not find claim at "{url}".')
|
||||
|
||||
|
||||
class IndexVersionMismatch(Exception):
|
||||
def __init__(self, got_version, expected_version):
|
||||
self.got_version = got_version
|
||||
self.expected_version = expected_version
|
||||
|
||||
|
||||
class SearchIndex:
|
||||
VERSION = 1
|
||||
|
||||
def __init__(self, index_prefix: str, search_timeout=3.0, elastic_host='localhost', elastic_port=9200):
|
||||
self.search_timeout = search_timeout
|
||||
self.sync_timeout = 600 # wont hit that 99% of the time, but can hit on a fresh import
|
||||
self.search_client: Optional[AsyncElasticsearch] = None
|
||||
self.sync_client: Optional[AsyncElasticsearch] = None
|
||||
self.index = index_prefix + 'claims'
|
||||
self.logger = class_logger(__name__, self.__class__.__name__)
|
||||
self.claim_cache = LRUCache(2 ** 15)
|
||||
self.short_id_cache = LRUCache(2 ** 17)
|
||||
self.search_cache = LRUCache(2 ** 17)
|
||||
self.resolution_cache = LRUCache(2 ** 17)
|
||||
self._elastic_host = elastic_host
|
||||
self._elastic_port = elastic_port
|
||||
|
||||
async def get_index_version(self) -> int:
|
||||
try:
|
||||
template = await self.sync_client.indices.get_template(self.index)
|
||||
return template[self.index]['version']
|
||||
except NotFoundError:
|
||||
return 0
|
||||
|
||||
async def set_index_version(self, version):
|
||||
await self.sync_client.indices.put_template(
|
||||
self.index, body={'version': version, 'index_patterns': ['ignored']}, ignore=400
|
||||
)
|
||||
|
||||
async def start(self) -> bool:
|
||||
if self.sync_client:
|
||||
return False
|
||||
hosts = [{'host': self._elastic_host, 'port': self._elastic_port}]
|
||||
self.sync_client = AsyncElasticsearch(hosts, timeout=self.sync_timeout)
|
||||
self.search_client = AsyncElasticsearch(hosts, timeout=self.search_timeout)
|
||||
while True:
|
||||
try:
|
||||
await self.sync_client.cluster.health(wait_for_status='yellow')
|
||||
break
|
||||
except ConnectionError:
|
||||
self.logger.warning("Failed to connect to Elasticsearch. Waiting for it!")
|
||||
await asyncio.sleep(1)
|
||||
|
||||
res = await self.sync_client.indices.create(self.index, INDEX_DEFAULT_SETTINGS, ignore=400)
|
||||
acked = res.get('acknowledged', False)
|
||||
if acked:
|
||||
await self.set_index_version(self.VERSION)
|
||||
return acked
|
||||
index_version = await self.get_index_version()
|
||||
if index_version != self.VERSION:
|
||||
self.logger.error("es search index has an incompatible version: %s vs %s", index_version, self.VERSION)
|
||||
raise IndexVersionMismatch(index_version, self.VERSION)
|
||||
return acked
|
||||
|
||||
def stop(self):
|
||||
clients = [self.sync_client, self.search_client]
|
||||
self.sync_client, self.search_client = None, None
|
||||
return asyncio.ensure_future(asyncio.gather(*(client.close() for client in clients)))
|
||||
|
||||
def delete_index(self):
|
||||
return self.sync_client.indices.delete(self.index, ignore_unavailable=True)
|
||||
|
||||
async def _consume_claim_producer(self, claim_producer):
|
||||
count = 0
|
||||
for op, doc in claim_producer:
|
||||
if op == 'delete':
|
||||
yield {'_index': self.index, '_op_type': 'delete', '_id': doc}
|
||||
else:
|
||||
yield extract_doc(doc, self.index)
|
||||
count += 1
|
||||
if count % 100 == 0:
|
||||
self.logger.info("Indexing in progress, %d claims.", count)
|
||||
self.logger.info("Indexing done for %d claims.", count)
|
||||
|
||||
async def claim_consumer(self, claim_producer):
|
||||
touched = set()
|
||||
async for ok, item in async_streaming_bulk(self.sync_client, self._consume_claim_producer(claim_producer),
|
||||
raise_on_error=False):
|
||||
if not ok:
|
||||
self.logger.warning("indexing failed for an item: %s", item)
|
||||
else:
|
||||
item = item.popitem()[1]
|
||||
touched.add(item['_id'])
|
||||
await self.sync_client.indices.refresh(self.index)
|
||||
self.logger.info("Indexing done.")
|
||||
|
||||
def update_filter_query(self, censor_type, blockdict, channels=False):
|
||||
blockdict = {key[::-1].hex(): value[::-1].hex() for key, value in blockdict.items()}
|
||||
if channels:
|
||||
update = expand_query(channel_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}")
|
||||
else:
|
||||
update = expand_query(claim_id__in=list(blockdict.keys()), censor_type=f"<{censor_type}")
|
||||
key = 'channel_id' if channels else 'claim_id'
|
||||
update['script'] = {
|
||||
"source": f"ctx._source.censor_type={censor_type}; ctx._source.censoring_channel_hash=params[ctx._source.{key}]",
|
||||
"lang": "painless",
|
||||
"params": blockdict
|
||||
}
|
||||
return update
|
||||
|
||||
async def apply_filters(self, blocked_streams, blocked_channels, filtered_streams, filtered_channels):
|
||||
if filtered_streams:
|
||||
await self.sync_client.update_by_query(
|
||||
self.index, body=self.update_filter_query(Censor.SEARCH, filtered_streams), slices=4)
|
||||
await self.sync_client.indices.refresh(self.index)
|
||||
if filtered_channels:
|
||||
await self.sync_client.update_by_query(
|
||||
self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels), slices=4)
|
||||
await self.sync_client.indices.refresh(self.index)
|
||||
await self.sync_client.update_by_query(
|
||||
self.index, body=self.update_filter_query(Censor.SEARCH, filtered_channels, True), slices=4)
|
||||
await self.sync_client.indices.refresh(self.index)
|
||||
if blocked_streams:
|
||||
await self.sync_client.update_by_query(
|
||||
self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_streams), slices=4)
|
||||
await self.sync_client.indices.refresh(self.index)
|
||||
if blocked_channels:
|
||||
await self.sync_client.update_by_query(
|
||||
self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels), slices=4)
|
||||
await self.sync_client.indices.refresh(self.index)
|
||||
await self.sync_client.update_by_query(
|
||||
self.index, body=self.update_filter_query(Censor.RESOLVE, blocked_channels, True), slices=4)
|
||||
await self.sync_client.indices.refresh(self.index)
|
||||
self.clear_caches()
|
||||
|
||||
def clear_caches(self):
|
||||
self.search_cache.clear()
|
||||
self.short_id_cache.clear()
|
||||
self.claim_cache.clear()
|
||||
self.resolution_cache.clear()
|
||||
|
||||
async def session_query(self, query_name, kwargs):
|
||||
offset, total = kwargs.get('offset', 0) if isinstance(kwargs, dict) else 0, 0
|
||||
total_referenced = []
|
||||
if query_name == 'resolve':
|
||||
total_referenced, response, censor = await self.resolve(*kwargs)
|
||||
else:
|
||||
cache_item = ResultCacheItem.from_cache(str(kwargs), self.search_cache)
|
||||
if cache_item.result is not None:
|
||||
return cache_item.result
|
||||
async with cache_item.lock:
|
||||
if cache_item.result:
|
||||
return cache_item.result
|
||||
censor = Censor(Censor.SEARCH)
|
||||
if kwargs.get('no_totals'):
|
||||
response, offset, total = await self.search(**kwargs, censor_type=Censor.NOT_CENSORED)
|
||||
else:
|
||||
response, offset, total = await self.search(**kwargs)
|
||||
censor.apply(response)
|
||||
total_referenced.extend(response)
|
||||
if censor.censored:
|
||||
response, _, _ = await self.search(**kwargs, censor_type=Censor.NOT_CENSORED)
|
||||
total_referenced.extend(response)
|
||||
result = Outputs.to_base64(
|
||||
response, await self._get_referenced_rows(total_referenced), offset, total, censor
|
||||
)
|
||||
cache_item.result = result
|
||||
return result
|
||||
return Outputs.to_base64(response, await self._get_referenced_rows(total_referenced), offset, total, censor)
|
||||
|
||||
async def resolve(self, *urls):
|
||||
censor = Censor(Censor.RESOLVE)
|
||||
results = [await self.resolve_url(url) for url in urls]
|
||||
# just heat the cache
|
||||
await self.populate_claim_cache(*filter(lambda x: isinstance(x, str), results))
|
||||
results = [self._get_from_cache_or_error(url, result) for url, result in zip(urls, results)]
|
||||
|
||||
censored = [
|
||||
result if not isinstance(result, dict) or not censor.censor(result)
|
||||
else ResolveCensoredError(url, result['censoring_channel_hash'])
|
||||
for url, result in zip(urls, results)
|
||||
]
|
||||
return results, censored, censor
|
||||
|
||||
def _get_from_cache_or_error(self, url: str, resolution: Union[LookupError, StreamResolution, ChannelResolution]):
|
||||
cached = self.claim_cache.get(resolution)
|
||||
return cached or (resolution if isinstance(resolution, LookupError) else resolution.lookup_error(url))
|
||||
|
||||
async def get_many(self, *claim_ids):
|
||||
await self.populate_claim_cache(*claim_ids)
|
||||
return filter(None, map(self.claim_cache.get, claim_ids))
|
||||
|
||||
async def populate_claim_cache(self, *claim_ids):
|
||||
missing = [claim_id for claim_id in claim_ids if self.claim_cache.get(claim_id) is None]
|
||||
if missing:
|
||||
results = await self.search_client.mget(
|
||||
index=self.index, body={"ids": missing}
|
||||
)
|
||||
for result in expand_result(filter(lambda doc: doc['found'], results["docs"])):
|
||||
self.claim_cache.set(result['claim_id'], result)
|
||||
|
||||
async def full_id_from_short_id(self, name, short_id, channel_id=None):
|
||||
key = '#'.join((channel_id or '', name, short_id))
|
||||
if key not in self.short_id_cache:
|
||||
query = {'name': name, 'claim_id': short_id}
|
||||
if channel_id:
|
||||
query['channel_id'] = channel_id
|
||||
query['order_by'] = ['^channel_join']
|
||||
query['signature_valid'] = True
|
||||
else:
|
||||
query['order_by'] = '^creation_height'
|
||||
result, _, _ = await self.search(**query, limit=1)
|
||||
if len(result) == 1:
|
||||
result = result[0]['claim_id']
|
||||
self.short_id_cache[key] = result
|
||||
return self.short_id_cache.get(key, None)
|
||||
|
||||
async def search(self, **kwargs):
|
||||
if 'channel' in kwargs:
|
||||
kwargs['channel_id'] = await self.resolve_url(kwargs.pop('channel'))
|
||||
if not kwargs['channel_id'] or not isinstance(kwargs['channel_id'], str):
|
||||
return [], 0, 0
|
||||
try:
|
||||
return await self.search_ahead(**kwargs)
|
||||
except NotFoundError:
|
||||
return [], 0, 0
|
||||
return expand_result(result['hits']), 0, result.get('total', {}).get('value', 0)
|
||||
|
||||
async def search_ahead(self, **kwargs):
|
||||
# 'limit_claims_per_channel' case. Fetch 1000 results, reorder, slice, inflate and return
|
||||
per_channel_per_page = kwargs.pop('limit_claims_per_channel', 0) or 0
|
||||
remove_duplicates = kwargs.pop('remove_duplicates', False)
|
||||
page_size = kwargs.pop('limit', 10)
|
||||
offset = kwargs.pop('offset', 0)
|
||||
kwargs['limit'] = 1000
|
||||
cache_item = ResultCacheItem.from_cache(f"ahead{per_channel_per_page}{kwargs}", self.search_cache)
|
||||
if cache_item.result is not None:
|
||||
reordered_hits = cache_item.result
|
||||
else:
|
||||
async with cache_item.lock:
|
||||
if cache_item.result:
|
||||
reordered_hits = cache_item.result
|
||||
else:
|
||||
query = expand_query(**kwargs)
|
||||
search_hits = deque((await self.search_client.search(
|
||||
query, index=self.index, track_total_hits=False,
|
||||
_source_includes=['_id', 'channel_id', 'reposted_claim_id', 'creation_height']
|
||||
))['hits']['hits'])
|
||||
if remove_duplicates:
|
||||
search_hits = self.__remove_duplicates(search_hits)
|
||||
if per_channel_per_page > 0:
|
||||
reordered_hits = self.__search_ahead(search_hits, page_size, per_channel_per_page)
|
||||
else:
|
||||
reordered_hits = [(hit['_id'], hit['_source']['channel_id']) for hit in search_hits]
|
||||
cache_item.result = reordered_hits
|
||||
result = list(await self.get_many(*(claim_id for claim_id, _ in reordered_hits[offset:(offset + page_size)])))
|
||||
return result, 0, len(reordered_hits)
|
||||
|
||||
def __remove_duplicates(self, search_hits: deque) -> deque:
|
||||
known_ids = {} # claim_id -> (creation_height, hit_id), where hit_id is either reposted claim id or original
|
||||
dropped = set()
|
||||
for hit in search_hits:
|
||||
hit_height, hit_id = hit['_source']['creation_height'], hit['_source']['reposted_claim_id'] or hit['_id']
|
||||
if hit_id not in known_ids:
|
||||
known_ids[hit_id] = (hit_height, hit['_id'])
|
||||
else:
|
||||
previous_height, previous_id = known_ids[hit_id]
|
||||
if hit_height < previous_height:
|
||||
known_ids[hit_id] = (hit_height, hit['_id'])
|
||||
dropped.add(previous_id)
|
||||
else:
|
||||
dropped.add(hit['_id'])
|
||||
return deque(hit for hit in search_hits if hit['_id'] not in dropped)
|
||||
|
||||
def __search_ahead(self, search_hits: list, page_size: int, per_channel_per_page: int):
|
||||
reordered_hits = []
|
||||
channel_counters = Counter()
|
||||
next_page_hits_maybe_check_later = deque()
|
||||
while search_hits or next_page_hits_maybe_check_later:
|
||||
if reordered_hits and len(reordered_hits) % page_size == 0:
|
||||
channel_counters.clear()
|
||||
elif not reordered_hits:
|
||||
pass
|
||||
else:
|
||||
break # means last page was incomplete and we are left with bad replacements
|
||||
for _ in range(len(next_page_hits_maybe_check_later)):
|
||||
claim_id, channel_id = next_page_hits_maybe_check_later.popleft()
|
||||
if per_channel_per_page > 0 and channel_counters[channel_id] < per_channel_per_page:
|
||||
reordered_hits.append((claim_id, channel_id))
|
||||
channel_counters[channel_id] += 1
|
||||
else:
|
||||
next_page_hits_maybe_check_later.append((claim_id, channel_id))
|
||||
while search_hits:
|
||||
hit = search_hits.popleft()
|
||||
hit_id, hit_channel_id = hit['_id'], hit['_source']['channel_id']
|
||||
if hit_channel_id is None or per_channel_per_page <= 0:
|
||||
reordered_hits.append((hit_id, hit_channel_id))
|
||||
elif channel_counters[hit_channel_id] < per_channel_per_page:
|
||||
reordered_hits.append((hit_id, hit_channel_id))
|
||||
channel_counters[hit_channel_id] += 1
|
||||
if len(reordered_hits) % page_size == 0:
|
||||
break
|
||||
else:
|
||||
next_page_hits_maybe_check_later.append((hit_id, hit_channel_id))
|
||||
return reordered_hits
|
||||
|
||||
async def resolve_url(self, raw_url):
|
||||
if raw_url not in self.resolution_cache:
|
||||
self.resolution_cache[raw_url] = await self._resolve_url(raw_url)
|
||||
return self.resolution_cache[raw_url]
|
||||
|
||||
async def _resolve_url(self, raw_url):
|
||||
try:
|
||||
url = URL.parse(raw_url)
|
||||
except ValueError as e:
|
||||
return e
|
||||
|
||||
stream = LookupError(f'Could not find claim at "{raw_url}".')
|
||||
|
||||
channel_id = await self.resolve_channel_id(url)
|
||||
if isinstance(channel_id, LookupError):
|
||||
return channel_id
|
||||
stream = (await self.resolve_stream(url, channel_id if isinstance(channel_id, str) else None)) or stream
|
||||
if url.has_stream:
|
||||
return StreamResolution(stream)
|
||||
else:
|
||||
return ChannelResolution(channel_id)
|
||||
|
||||
async def resolve_channel_id(self, url: URL):
|
||||
if not url.has_channel:
|
||||
return
|
||||
if url.channel.is_fullid:
|
||||
return url.channel.claim_id
|
||||
if url.channel.is_shortid:
|
||||
channel_id = await self.full_id_from_short_id(url.channel.name, url.channel.claim_id)
|
||||
if not channel_id:
|
||||
return LookupError(f'Could not find channel in "{url}".')
|
||||
return channel_id
|
||||
|
||||
query = url.channel.to_dict()
|
||||
if set(query) == {'name'}:
|
||||
query['is_controlling'] = True
|
||||
else:
|
||||
query['order_by'] = ['^creation_height']
|
||||
matches, _, _ = await self.search(**query, limit=1)
|
||||
if matches:
|
||||
channel_id = matches[0]['claim_id']
|
||||
else:
|
||||
return LookupError(f'Could not find channel in "{url}".')
|
||||
return channel_id
|
||||
|
||||
async def resolve_stream(self, url: URL, channel_id: str = None):
|
||||
if not url.has_stream:
|
||||
return None
|
||||
if url.has_channel and channel_id is None:
|
||||
return None
|
||||
query = url.stream.to_dict()
|
||||
if url.stream.claim_id is not None:
|
||||
if url.stream.is_fullid:
|
||||
claim_id = url.stream.claim_id
|
||||
else:
|
||||
claim_id = await self.full_id_from_short_id(query['name'], query['claim_id'], channel_id)
|
||||
return claim_id
|
||||
|
||||
if channel_id is not None:
|
||||
if set(query) == {'name'}:
|
||||
# temporarily emulate is_controlling for claims in channel
|
||||
query['order_by'] = ['effective_amount', '^height']
|
||||
else:
|
||||
query['order_by'] = ['^channel_join']
|
||||
query['channel_id'] = channel_id
|
||||
query['signature_valid'] = True
|
||||
elif set(query) == {'name'}:
|
||||
query['is_controlling'] = True
|
||||
matches, _, _ = await self.search(**query, limit=1)
|
||||
if matches:
|
||||
return matches[0]['claim_id']
|
||||
|
||||
async def _get_referenced_rows(self, txo_rows: List[dict]):
|
||||
txo_rows = [row for row in txo_rows if isinstance(row, dict)]
|
||||
referenced_ids = set(filter(None, map(itemgetter('reposted_claim_id'), txo_rows)))
|
||||
referenced_ids |= set(filter(None, (row['channel_id'] for row in txo_rows)))
|
||||
referenced_ids |= set(map(parse_claim_id, filter(None, (row['censoring_channel_hash'] for row in txo_rows))))
|
||||
|
||||
referenced_txos = []
|
||||
if referenced_ids:
|
||||
referenced_txos.extend(await self.get_many(*referenced_ids))
|
||||
referenced_ids = set(filter(None, (row['channel_id'] for row in referenced_txos)))
|
||||
|
||||
if referenced_ids:
|
||||
referenced_txos.extend(await self.get_many(*referenced_ids))
|
||||
|
||||
return referenced_txos
|
||||
|
||||
|
||||
def extract_doc(doc, index):
|
||||
doc['claim_id'] = doc.pop('claim_hash')[::-1].hex()
|
||||
if doc['reposted_claim_hash'] is not None:
|
||||
doc['reposted_claim_id'] = doc.pop('reposted_claim_hash')[::-1].hex()
|
||||
else:
|
||||
doc['reposted_claim_id'] = None
|
||||
channel_hash = doc.pop('channel_hash')
|
||||
doc['channel_id'] = channel_hash[::-1].hex() if channel_hash else channel_hash
|
||||
channel_hash = doc.pop('censoring_channel_hash')
|
||||
doc['censoring_channel_hash'] = channel_hash[::-1].hex() if channel_hash else channel_hash
|
||||
txo_hash = doc.pop('txo_hash')
|
||||
doc['tx_id'] = txo_hash[:32][::-1].hex()
|
||||
doc['tx_nout'] = struct.unpack('<I', txo_hash[32:])[0]
|
||||
doc['is_controlling'] = bool(doc['is_controlling'])
|
||||
doc['signature'] = (doc.pop('signature') or b'').hex() or None
|
||||
doc['signature_digest'] = (doc.pop('signature_digest') or b'').hex() or None
|
||||
doc['public_key_bytes'] = (doc.pop('public_key_bytes') or b'').hex() or None
|
||||
doc['public_key_hash'] = (doc.pop('public_key_hash') or b'').hex() or None
|
||||
doc['signature_valid'] = bool(doc['signature_valid'])
|
||||
doc['claim_type'] = doc.get('claim_type', 0) or 0
|
||||
doc['stream_type'] = int(doc.get('stream_type', 0) or 0)
|
||||
doc['has_source'] = bool(doc['has_source'])
|
||||
return {'doc': doc, '_id': doc['claim_id'], '_index': index, '_op_type': 'update', 'doc_as_upsert': True}
|
||||
|
||||
|
||||
def expand_query(**kwargs):
|
||||
if "amount_order" in kwargs:
|
||||
kwargs["limit"] = 1
|
||||
kwargs["order_by"] = "effective_amount"
|
||||
kwargs["offset"] = int(kwargs["amount_order"]) - 1
|
||||
if 'name' in kwargs:
|
||||
kwargs['name'] = normalize_name(kwargs.pop('name'))
|
||||
if kwargs.get('is_controlling') is False:
|
||||
kwargs.pop('is_controlling')
|
||||
query = {'must': [], 'must_not': []}
|
||||
collapse = None
|
||||
for key, value in kwargs.items():
|
||||
key = key.replace('claim.', '')
|
||||
many = key.endswith('__in') or isinstance(value, list)
|
||||
if many:
|
||||
key = key.replace('__in', '')
|
||||
value = list(filter(None, value))
|
||||
if value is None or isinstance(value, list) and len(value) == 0:
|
||||
continue
|
||||
key = REPLACEMENTS.get(key, key)
|
||||
if key in FIELDS:
|
||||
partial_id = False
|
||||
if key == 'claim_type':
|
||||
if isinstance(value, str):
|
||||
value = CLAIM_TYPES[value]
|
||||
else:
|
||||
value = [CLAIM_TYPES[claim_type] for claim_type in value]
|
||||
if key == '_id':
|
||||
if isinstance(value, Iterable):
|
||||
value = [item[::-1].hex() for item in value]
|
||||
else:
|
||||
value = value[::-1].hex()
|
||||
if not many and key in ('_id', 'claim_id') and len(value) < 20:
|
||||
partial_id = True
|
||||
if key == 'public_key_id':
|
||||
key = 'public_key_hash'
|
||||
value = Base58.decode(value)[1:21].hex()
|
||||
if key == 'signature_valid':
|
||||
continue # handled later
|
||||
if key in TEXT_FIELDS:
|
||||
key += '.keyword'
|
||||
ops = {'<=': 'lte', '>=': 'gte', '<': 'lt', '>': 'gt'}
|
||||
if partial_id:
|
||||
query['must'].append({"prefix": {"claim_id": value}})
|
||||
elif key in RANGE_FIELDS and isinstance(value, str) and value[0] in ops:
|
||||
operator_length = 2 if value[:2] in ops else 1
|
||||
operator, value = value[:operator_length], value[operator_length:]
|
||||
if key == 'fee_amount':
|
||||
value = str(Decimal(value)*1000)
|
||||
query['must'].append({"range": {key: {ops[operator]: value}}})
|
||||
elif many:
|
||||
query['must'].append({"terms": {key: value}})
|
||||
else:
|
||||
if key == 'fee_amount':
|
||||
value = str(Decimal(value)*1000)
|
||||
query['must'].append({"term": {key: {"value": value}}})
|
||||
elif key == 'not_channel_ids':
|
||||
for channel_id in value:
|
||||
query['must_not'].append({"term": {'channel_id.keyword': channel_id}})
|
||||
query['must_not'].append({"term": {'_id': channel_id}})
|
||||
elif key == 'channel_ids':
|
||||
query['must'].append({"terms": {'channel_id.keyword': value}})
|
||||
elif key == 'claim_ids':
|
||||
query['must'].append({"terms": {'claim_id.keyword': value}})
|
||||
elif key == 'media_types':
|
||||
query['must'].append({"terms": {'media_type.keyword': value}})
|
||||
elif key == 'stream_types':
|
||||
query['must'].append({"terms": {'stream_type': [STREAM_TYPES[stype] for stype in value]}})
|
||||
elif key == 'any_languages':
|
||||
query['must'].append({"terms": {'languages': clean_tags(value)}})
|
||||
elif key == 'any_languages':
|
||||
query['must'].append({"terms": {'languages': value}})
|
||||
elif key == 'all_languages':
|
||||
query['must'].extend([{"term": {'languages': tag}} for tag in value])
|
||||
elif key == 'any_tags':
|
||||
query['must'].append({"terms": {'tags.keyword': clean_tags(value)}})
|
||||
elif key == 'all_tags':
|
||||
query['must'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)])
|
||||
elif key == 'not_tags':
|
||||
query['must_not'].extend([{"term": {'tags.keyword': tag}} for tag in clean_tags(value)])
|
||||
elif key == 'not_claim_id':
|
||||
query['must_not'].extend([{"term": {'claim_id.keyword': cid}} for cid in value])
|
||||
elif key == 'limit_claims_per_channel':
|
||||
collapse = ('channel_id.keyword', value)
|
||||
if kwargs.get('has_channel_signature'):
|
||||
query['must'].append({"exists": {"field": "signature_digest"}})
|
||||
if 'signature_valid' in kwargs:
|
||||
query['must'].append({"term": {"signature_valid": bool(kwargs["signature_valid"])}})
|
||||
elif 'signature_valid' in kwargs:
|
||||
query.setdefault('should', [])
|
||||
query["minimum_should_match"] = 1
|
||||
query['should'].append({"bool": {"must_not": {"exists": {"field": "signature_digest"}}}})
|
||||
query['should'].append({"term": {"signature_valid": bool(kwargs["signature_valid"])}})
|
||||
if 'has_source' in kwargs:
|
||||
query.setdefault('should', [])
|
||||
query["minimum_should_match"] = 1
|
||||
is_stream_or_repost = {"terms": {"claim_type": [CLAIM_TYPES['stream'], CLAIM_TYPES['repost']]}}
|
||||
query['should'].append(
|
||||
{"bool": {"must": [{"match": {"has_source": kwargs['has_source']}}, is_stream_or_repost]}})
|
||||
query['should'].append({"bool": {"must_not": [is_stream_or_repost]}})
|
||||
query['should'].append({"bool": {"must": [{"term": {"reposted_claim_type": CLAIM_TYPES['channel']}}]}})
|
||||
if kwargs.get('text'):
|
||||
query['must'].append(
|
||||
{"simple_query_string":
|
||||
{"query": kwargs["text"], "fields": [
|
||||
"claim_name^4", "channel_name^8", "title^1", "description^.5", "author^1", "tags^.5"
|
||||
]}})
|
||||
query = {
|
||||
"_source": {"excludes": ["description", "title"]},
|
||||
'query': {'bool': query},
|
||||
"sort": [],
|
||||
}
|
||||
if "limit" in kwargs:
|
||||
query["size"] = kwargs["limit"]
|
||||
if 'offset' in kwargs:
|
||||
query["from"] = kwargs["offset"]
|
||||
if 'order_by' in kwargs:
|
||||
if isinstance(kwargs["order_by"], str):
|
||||
kwargs["order_by"] = [kwargs["order_by"]]
|
||||
for value in kwargs['order_by']:
|
||||
if 'trending_group' in value:
|
||||
# fixme: trending_mixed is 0 for all records on variable decay, making sort slow.
|
||||
continue
|
||||
is_asc = value.startswith('^')
|
||||
value = value[1:] if is_asc else value
|
||||
value = REPLACEMENTS.get(value, value)
|
||||
if value in TEXT_FIELDS:
|
||||
value += '.keyword'
|
||||
query['sort'].append({value: "asc" if is_asc else "desc"})
|
||||
if collapse:
|
||||
query["collapse"] = {
|
||||
"field": collapse[0],
|
||||
"inner_hits": {
|
||||
"name": collapse[0],
|
||||
"size": collapse[1],
|
||||
"sort": query["sort"]
|
||||
}
|
||||
}
|
||||
return query
|
||||
|
||||
|
||||
def expand_result(results):
|
||||
inner_hits = []
|
||||
expanded = []
|
||||
for result in results:
|
||||
if result.get("inner_hits"):
|
||||
for _, inner_hit in result["inner_hits"].items():
|
||||
inner_hits.extend(inner_hit["hits"]["hits"])
|
||||
continue
|
||||
result = result['_source']
|
||||
result['claim_hash'] = unhexlify(result['claim_id'])[::-1]
|
||||
if result['reposted_claim_id']:
|
||||
result['reposted_claim_hash'] = unhexlify(result['reposted_claim_id'])[::-1]
|
||||
else:
|
||||
result['reposted_claim_hash'] = None
|
||||
result['channel_hash'] = unhexlify(result['channel_id'])[::-1] if result['channel_id'] else None
|
||||
result['txo_hash'] = unhexlify(result['tx_id'])[::-1] + struct.pack('<I', result['tx_nout'])
|
||||
result['tx_hash'] = unhexlify(result['tx_id'])[::-1]
|
||||
if result['censoring_channel_hash']:
|
||||
result['censoring_channel_hash'] = unhexlify(result['censoring_channel_hash'])[::-1]
|
||||
expanded.append(result)
|
||||
if inner_hits:
|
||||
return expand_result(inner_hits)
|
||||
return expanded
|
||||
|
||||
|
||||
class ResultCacheItem:
|
||||
__slots__ = '_result', 'lock', 'has_result'
|
||||
|
||||
def __init__(self):
|
||||
self.has_result = asyncio.Event()
|
||||
self.lock = asyncio.Lock()
|
||||
self._result = None
|
||||
|
||||
@property
|
||||
def result(self) -> str:
|
||||
return self._result
|
||||
|
||||
@result.setter
|
||||
def result(self, result: str):
|
||||
self._result = result
|
||||
if result is not None:
|
||||
self.has_result.set()
|
||||
|
||||
@classmethod
|
||||
def from_cache(cls, cache_key, cache):
|
||||
cache_item = cache.get(cache_key)
|
||||
if cache_item is None:
|
||||
cache_item = cache[cache_key] = ResultCacheItem()
|
||||
return cache_item
|
||||
@@ -1,122 +0,0 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from collections import namedtuple
|
||||
from multiprocessing import Process
|
||||
|
||||
import apsw
|
||||
from elasticsearch import AsyncElasticsearch
|
||||
from elasticsearch.helpers import async_bulk
|
||||
from lbry.wallet.server.env import Env
|
||||
from lbry.wallet.server.coin import LBC
|
||||
from lbry.wallet.server.db.elasticsearch.search import extract_doc, SearchIndex, IndexVersionMismatch
|
||||
|
||||
|
||||
async def get_all(db, shard_num, shards_total, limit=0, index_name='claims'):
|
||||
logging.info("shard %d starting", shard_num)
|
||||
def exec_factory(cursor, statement, bindings):
|
||||
tpl = namedtuple('row', (d[0] for d in cursor.getdescription()))
|
||||
cursor.setrowtrace(lambda cursor, row: tpl(*row))
|
||||
return True
|
||||
|
||||
db.setexectrace(exec_factory)
|
||||
total = db.execute(f"select count(*) as total from claim where height % {shards_total} = {shard_num};").fetchone()[0]
|
||||
for num, claim in enumerate(db.execute(f"""
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
|
||||
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash)
|
||||
WHERE claim.height % {shards_total} = {shard_num}
|
||||
ORDER BY claim.height desc
|
||||
""")):
|
||||
claim = dict(claim._asdict())
|
||||
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
|
||||
claim['censor_type'] = 0
|
||||
claim['censoring_channel_hash'] = None
|
||||
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
|
||||
claim['languages'] = claim['languages'].split(' ') if claim['languages'] else []
|
||||
if num % 10_000 == 0:
|
||||
logging.info("%d/%d", num, total)
|
||||
yield extract_doc(claim, index_name)
|
||||
if 0 < limit <= num:
|
||||
break
|
||||
|
||||
|
||||
async def consume(producer, index_name):
|
||||
env = Env(LBC)
|
||||
logging.info("ES sync host: %s:%i", env.elastic_host, env.elastic_port)
|
||||
es = AsyncElasticsearch([{'host': env.elastic_host, 'port': env.elastic_port}])
|
||||
try:
|
||||
await async_bulk(es, producer, request_timeout=120)
|
||||
await es.indices.refresh(index=index_name)
|
||||
finally:
|
||||
await es.close()
|
||||
|
||||
|
||||
async def make_es_index(index=None):
|
||||
env = Env(LBC)
|
||||
if index is None:
|
||||
index = SearchIndex('', elastic_host=env.elastic_host, elastic_port=env.elastic_port)
|
||||
|
||||
try:
|
||||
return await index.start()
|
||||
except IndexVersionMismatch as err:
|
||||
logging.info(
|
||||
"dropping ES search index (version %s) for upgrade to version %s", err.got_version, err.expected_version
|
||||
)
|
||||
await index.delete_index()
|
||||
await index.stop()
|
||||
return await index.start()
|
||||
finally:
|
||||
index.stop()
|
||||
|
||||
|
||||
async def run(db_path, clients, blocks, shard, index_name='claims'):
|
||||
def itsbusy(*_):
|
||||
logging.info("shard %d: db is busy, retry", shard)
|
||||
return True
|
||||
db = apsw.Connection(db_path, flags=apsw.SQLITE_OPEN_READONLY | apsw.SQLITE_OPEN_URI)
|
||||
db.setbusyhandler(itsbusy)
|
||||
db.cursor().execute('pragma journal_mode=wal;')
|
||||
db.cursor().execute('pragma temp_store=memory;')
|
||||
|
||||
producer = get_all(db.cursor(), shard, clients, limit=blocks, index_name=index_name)
|
||||
await asyncio.gather(*(consume(producer, index_name=index_name) for _ in range(min(8, clients))))
|
||||
|
||||
|
||||
def __run(args, shard):
|
||||
asyncio.run(run(args.db_path, args.clients, args.blocks, shard))
|
||||
|
||||
|
||||
def run_elastic_sync():
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logging.getLogger('aiohttp').setLevel(logging.WARNING)
|
||||
logging.getLogger('elasticsearch').setLevel(logging.WARNING)
|
||||
|
||||
logging.info('lbry.server starting')
|
||||
parser = argparse.ArgumentParser(prog="lbry-hub-elastic-sync")
|
||||
parser.add_argument("db_path", type=str)
|
||||
parser.add_argument("-c", "--clients", type=int, default=16)
|
||||
parser.add_argument("-b", "--blocks", type=int, default=0)
|
||||
parser.add_argument("-f", "--force", default=False, action='store_true')
|
||||
args = parser.parse_args()
|
||||
processes = []
|
||||
|
||||
if not args.force and not os.path.exists(args.db_path):
|
||||
logging.info("DB path doesnt exist")
|
||||
return
|
||||
|
||||
if not args.force and not asyncio.run(make_es_index()):
|
||||
logging.info("ES is already initialized")
|
||||
return
|
||||
for i in range(args.clients):
|
||||
processes.append(Process(target=__run, args=(args, i)))
|
||||
processes[-1].start()
|
||||
for process in processes:
|
||||
process.join()
|
||||
process.close()
|
||||
@@ -0,0 +1,52 @@
|
||||
from lbry.wallet.database import constraints_to_sql
|
||||
|
||||
CREATE_FULL_TEXT_SEARCH = """
|
||||
create virtual table if not exists search using fts5(
|
||||
claim_name, channel_name, title, description, author, tags,
|
||||
content=claim, tokenize=porter
|
||||
);
|
||||
"""
|
||||
|
||||
FTS_ORDER_BY = "bm25(search, 4.0, 8.0, 1.0, 0.5, 1.0, 0.5)"
|
||||
|
||||
|
||||
def fts_action_sql(claims=None, action='insert'):
|
||||
select = {
|
||||
'rowid': "claim.rowid",
|
||||
'claim_name': "claim.normalized",
|
||||
'channel_name': "channel.normalized",
|
||||
'title': "claim.title",
|
||||
'description': "claim.description",
|
||||
'author': "claim.author",
|
||||
'tags': "(select group_concat(tag, ' ') from tag where tag.claim_hash=claim.claim_hash)"
|
||||
}
|
||||
if action == 'delete':
|
||||
select['search'] = '"delete"'
|
||||
|
||||
where, values = "", {}
|
||||
if claims:
|
||||
where, values = constraints_to_sql({'claim.claim_hash__in': claims})
|
||||
where = 'WHERE '+where
|
||||
|
||||
return f"""
|
||||
INSERT INTO search ({','.join(select.keys())})
|
||||
SELECT {','.join(select.values())} FROM claim
|
||||
LEFT JOIN claim as channel ON (claim.channel_hash=channel.claim_hash) {where}
|
||||
""", values
|
||||
|
||||
|
||||
def update_full_text_search(action, outputs, db, is_first_sync):
|
||||
if is_first_sync:
|
||||
return
|
||||
if not outputs:
|
||||
return
|
||||
if action in ("before-delete", "before-update"):
|
||||
db.execute(*fts_action_sql(outputs, 'delete'))
|
||||
elif action in ("after-insert", "after-update"):
|
||||
db.execute(*fts_action_sql(outputs, 'insert'))
|
||||
else:
|
||||
raise ValueError(f"Invalid action for updating full text search: '{action}'")
|
||||
|
||||
|
||||
def first_sync_finished(db):
|
||||
db.execute(*fts_action_sql())
|
||||
@@ -8,17 +8,17 @@ from binascii import unhexlify
|
||||
from decimal import Decimal
|
||||
from contextvars import ContextVar
|
||||
from functools import wraps
|
||||
from itertools import chain
|
||||
from dataclasses import dataclass
|
||||
|
||||
from lbry.wallet.database import query, interpolate
|
||||
from lbry.error import ResolveCensoredError
|
||||
|
||||
from lbry.schema.url import URL, normalize_name
|
||||
from lbry.schema.tags import clean_tags
|
||||
from lbry.schema.result import Outputs, Censor
|
||||
from lbry.wallet import Ledger, RegTestLedger
|
||||
|
||||
from lbry.wallet.server.db.common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
from .full_text_search import FTS_ORDER_BY
|
||||
|
||||
|
||||
class SQLiteOperationalError(apsw.Error):
|
||||
@@ -37,8 +37,8 @@ ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
|
||||
INTEGER_PARAMS = {
|
||||
'height', 'creation_height', 'activation_height', 'expiration_height',
|
||||
'timestamp', 'creation_timestamp', 'duration', 'release_time', 'fee_amount',
|
||||
'tx_position', 'channel_join', 'reposted', 'limit_claims_per_channel',
|
||||
'timestamp', 'creation_timestamp', 'release_time', 'fee_amount',
|
||||
'tx_position', 'channel_join', 'reposted',
|
||||
'amount', 'effective_amount', 'support_amount',
|
||||
'trending_group', 'trending_mixed',
|
||||
'trending_local', 'trending_global',
|
||||
@@ -52,7 +52,7 @@ SEARCH_PARAMS = {
|
||||
'any_locations', 'all_locations', 'not_locations',
|
||||
'any_languages', 'all_languages', 'not_languages',
|
||||
'is_controlling', 'limit', 'offset', 'order_by',
|
||||
'no_totals', 'has_source'
|
||||
'no_totals',
|
||||
} | INTEGER_PARAMS
|
||||
|
||||
|
||||
@@ -93,10 +93,10 @@ class ReaderState:
|
||||
self.db.setprogresshandler(interruptor, 100)
|
||||
|
||||
def get_resolve_censor(self) -> Censor:
|
||||
return Censor(Censor.RESOLVE)
|
||||
return Censor(self.blocked_streams, self.blocked_channels)
|
||||
|
||||
def get_search_censor(self, limit_claims_per_channel: int) -> Censor:
|
||||
return Censor(Censor.SEARCH)
|
||||
def get_search_censor(self) -> Censor:
|
||||
return Censor(self.filtered_streams, self.filtered_channels)
|
||||
|
||||
|
||||
ctx: ContextVar[Optional[ReaderState]] = ContextVar('ctx')
|
||||
@@ -265,7 +265,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
else:
|
||||
constraints['claim.claim_id__like'] = f'{claim_id[:40]}%'
|
||||
elif 'claim_ids' in constraints:
|
||||
constraints['claim.claim_id__in'] = set(constraints.pop('claim_ids'))
|
||||
constraints['claim.claim_id__in'] = constraints.pop('claim_ids')
|
||||
|
||||
if 'reposted_claim_id' in constraints:
|
||||
constraints['claim.reposted_claim_hash'] = unhexlify(constraints.pop('reposted_claim_id'))[::-1]
|
||||
@@ -281,16 +281,15 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
if 'channel_ids' in constraints:
|
||||
channel_ids = constraints.pop('channel_ids')
|
||||
if channel_ids:
|
||||
constraints['claim.channel_hash__in'] = {
|
||||
unhexlify(cid)[::-1] for cid in channel_ids if cid
|
||||
}
|
||||
constraints['claim.channel_hash__in'] = [
|
||||
unhexlify(cid)[::-1] for cid in channel_ids
|
||||
]
|
||||
if 'not_channel_ids' in constraints:
|
||||
not_channel_ids = constraints.pop('not_channel_ids')
|
||||
if not_channel_ids:
|
||||
not_channel_ids_binary = {
|
||||
not_channel_ids_binary = [
|
||||
unhexlify(ncid)[::-1] for ncid in not_channel_ids
|
||||
}
|
||||
constraints['claim.claim_hash__not_in#not_channel_ids'] = not_channel_ids_binary
|
||||
]
|
||||
if constraints.get('has_channel_signature', False):
|
||||
constraints['claim.channel_hash__not_in'] = not_channel_ids_binary
|
||||
else:
|
||||
@@ -316,23 +315,17 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
constraints['claim.txo_hash'] = tx_hash + struct.pack('<I', nout)
|
||||
|
||||
if 'claim_type' in constraints:
|
||||
claim_types = constraints.pop('claim_type')
|
||||
if isinstance(claim_types, str):
|
||||
claim_types = [claim_types]
|
||||
if claim_types:
|
||||
constraints['claim.claim_type__in'] = {
|
||||
CLAIM_TYPES[claim_type] for claim_type in claim_types
|
||||
}
|
||||
constraints['claim.claim_type'] = CLAIM_TYPES[constraints.pop('claim_type')]
|
||||
if 'stream_types' in constraints:
|
||||
stream_types = constraints.pop('stream_types')
|
||||
if stream_types:
|
||||
constraints['claim.stream_type__in'] = {
|
||||
constraints['claim.stream_type__in'] = [
|
||||
STREAM_TYPES[stream_type] for stream_type in stream_types
|
||||
}
|
||||
]
|
||||
if 'media_types' in constraints:
|
||||
media_types = constraints.pop('media_types')
|
||||
if media_types:
|
||||
constraints['claim.media_type__in'] = set(media_types)
|
||||
constraints['claim.media_type__in'] = media_types
|
||||
|
||||
if 'fee_currency' in constraints:
|
||||
constraints['claim.fee_currency'] = constraints.pop('fee_currency').lower()
|
||||
@@ -341,7 +334,12 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
_apply_constraints_for_array_attributes(constraints, 'language', lambda _: _, for_count)
|
||||
_apply_constraints_for_array_attributes(constraints, 'location', lambda _: _, for_count)
|
||||
|
||||
select = f"SELECT {cols} FROM claim"
|
||||
if 'text' in constraints:
|
||||
constraints["search"] = constraints.pop("text")
|
||||
constraints["order_by"] = FTS_ORDER_BY
|
||||
select = f"SELECT {cols} FROM search JOIN claim ON (search.rowid=claim.rowid)"
|
||||
else:
|
||||
select = f"SELECT {cols} FROM claim"
|
||||
if not for_count:
|
||||
select += " LEFT JOIN claimtrie USING (claim_hash)"
|
||||
return query(select, **constraints)
|
||||
@@ -366,7 +364,7 @@ def count_claims(**constraints) -> int:
|
||||
constraints.pop('offset', None)
|
||||
constraints.pop('limit', None)
|
||||
constraints.pop('order_by', None)
|
||||
count = select_claims(Censor(Censor.SEARCH), 'count(*) as row_count', for_count=True, **constraints)
|
||||
count = select_claims(Censor(), 'count(*) as row_count', for_count=True, **constraints)
|
||||
return count[0]['row_count']
|
||||
|
||||
|
||||
@@ -390,13 +388,9 @@ def search_claims(censor: Censor, **constraints) -> List:
|
||||
)
|
||||
|
||||
|
||||
def _get_referenced_rows(txo_rows: List[dict], censor_channels: List[bytes]):
|
||||
censor = ctx.get().get_resolve_censor()
|
||||
def _get_referenced_rows(censor: Censor, txo_rows: List[dict]):
|
||||
repost_hashes = set(filter(None, map(itemgetter('reposted_claim_hash'), txo_rows)))
|
||||
channel_hashes = set(chain(
|
||||
filter(None, map(itemgetter('channel_hash'), txo_rows)),
|
||||
censor_channels
|
||||
))
|
||||
channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows)))
|
||||
|
||||
reposted_txos = []
|
||||
if repost_hashes:
|
||||
@@ -410,20 +404,22 @@ def _get_referenced_rows(txo_rows: List[dict], censor_channels: List[bytes]):
|
||||
# channels must come first for client side inflation to work properly
|
||||
return channel_txos + reposted_txos
|
||||
|
||||
|
||||
@measure
|
||||
def search(constraints) -> Tuple[List, List, int, int, Censor]:
|
||||
assert set(constraints).issubset(SEARCH_PARAMS), \
|
||||
f"Search query contains invalid arguments: {set(constraints).difference(SEARCH_PARAMS)}"
|
||||
total = None
|
||||
limit_claims_per_channel = constraints.pop('limit_claims_per_channel', None)
|
||||
if not constraints.pop('no_totals', False):
|
||||
total = count_claims(**constraints)
|
||||
constraints['offset'] = abs(constraints.get('offset', 0))
|
||||
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = ["claim_hash"]
|
||||
context = ctx.get()
|
||||
search_censor = context.get_search_censor(limit_claims_per_channel)
|
||||
search_censor = context.get_search_censor()
|
||||
txo_rows = search_claims(search_censor, **constraints)
|
||||
extra_txo_rows = _get_referenced_rows(txo_rows, search_censor.censored.keys())
|
||||
extra_txo_rows = _get_referenced_rows(context.get_resolve_censor(), txo_rows)
|
||||
return txo_rows, extra_txo_rows, constraints['offset'], total, search_censor
|
||||
|
||||
|
||||
@@ -431,8 +427,7 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
|
||||
def resolve(urls) -> Tuple[List, List]:
|
||||
txo_rows = [resolve_url(raw_url) for raw_url in urls]
|
||||
extra_txo_rows = _get_referenced_rows(
|
||||
[txo for txo in txo_rows if isinstance(txo, dict)],
|
||||
[txo.censor_hash for txo in txo_rows if isinstance(txo, ResolveCensoredError)]
|
||||
ctx.get().get_resolve_censor(), [r for r in txo_rows if isinstance(r, dict)]
|
||||
)
|
||||
return txo_rows, extra_txo_rows
|
||||
|
||||
@@ -457,8 +452,6 @@ def resolve_url(raw_url):
|
||||
matches = search_claims(censor, **query, limit=1)
|
||||
if matches:
|
||||
channel = matches[0]
|
||||
elif censor.censored:
|
||||
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
|
||||
else:
|
||||
return LookupError(f'Could not find channel in "{raw_url}".')
|
||||
|
||||
@@ -477,22 +470,12 @@ def resolve_url(raw_url):
|
||||
matches = search_claims(censor, **query, limit=1)
|
||||
if matches:
|
||||
return matches[0]
|
||||
elif censor.censored:
|
||||
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
|
||||
else:
|
||||
return LookupError(f'Could not find claim at "{raw_url}".')
|
||||
return LookupError(f'Could not find stream in "{raw_url}".')
|
||||
|
||||
return channel
|
||||
|
||||
|
||||
CLAIM_HASH_OR_REPOST_HASH_SQL = f"""
|
||||
CASE WHEN claim.claim_type = {CLAIM_TYPES['repost']}
|
||||
THEN claim.reposted_claim_hash
|
||||
ELSE claim.claim_hash
|
||||
END
|
||||
"""
|
||||
|
||||
|
||||
def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_count=False):
|
||||
any_items = set(cleaner(constraints.pop(f'any_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
all_items = set(cleaner(constraints.pop(f'all_{attr}s', []))[:ATTRIBUTE_ARRAY_MAX_LENGTH])
|
||||
@@ -512,8 +495,7 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
index_name = COMMON_TAGS[item]
|
||||
any_queries[f'#_common_tag_{index_name}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx
|
||||
WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
|
||||
SELECT 1 FROM tag INDEXED BY tag_{index_name}_idx WHERE claim.claim_hash=tag.claim_hash
|
||||
AND tag = '{item}'
|
||||
)
|
||||
"""
|
||||
@@ -526,22 +508,10 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
)
|
||||
any_queries[f'#_any_common_tags'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM tag WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=tag.claim_hash
|
||||
SELECT 1 FROM tag WHERE claim.claim_hash=tag.claim_hash
|
||||
AND tag IN ({values})
|
||||
)
|
||||
"""
|
||||
elif attr == 'language':
|
||||
indexed_languages = any_items & set(INDEXED_LANGUAGES)
|
||||
if indexed_languages:
|
||||
any_items -= indexed_languages
|
||||
for language in indexed_languages:
|
||||
any_queries[f'#_any_common_languages_{language}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM language INDEXED BY language_{language}_idx
|
||||
WHERE {CLAIM_HASH_OR_REPOST_HASH_SQL}=language.claim_hash
|
||||
AND language = '{language}'
|
||||
)
|
||||
"""
|
||||
|
||||
if any_items:
|
||||
|
||||
@@ -552,24 +522,14 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
f':$any_{attr}{i}' for i in range(len(any_items))
|
||||
)
|
||||
if for_count or attr == 'tag':
|
||||
if attr == 'tag':
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
((claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
|
||||
(claim.claim_type == {CLAIM_TYPES['repost']} AND
|
||||
claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
any_queries[f'claim.claim_hash__in#_any_{attr}'] = f"""
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
"""
|
||||
else:
|
||||
any_queries[f'#_any_{attr}'] = f"""
|
||||
EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
@@ -588,17 +548,15 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
f':$all_{attr}{i}' for i in range(len(all_items))
|
||||
)
|
||||
if for_count:
|
||||
constraints[f'#_all_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} IN (
|
||||
constraints[f'claim.claim_hash__in#_all_{attr}'] = f"""
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
GROUP BY claim_hash HAVING COUNT({attr}) = :$all_{attr}_count
|
||||
)
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_all_{attr}'] = f"""
|
||||
{len(all_items)}=(
|
||||
SELECT count(*) FROM {attr} WHERE
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
@@ -611,24 +569,14 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun
|
||||
f':$not_{attr}{i}' for i in range(len(not_items))
|
||||
)
|
||||
if for_count:
|
||||
if attr == 'tag':
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
((claim.claim_type != {CLAIM_TYPES['repost']}
|
||||
AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR
|
||||
(claim.claim_type == {CLAIM_TYPES['repost']} AND
|
||||
claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))))
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL} NOT IN (
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
constraints[f'claim.claim_hash__not_in#_not_{attr}'] = f"""
|
||||
SELECT claim_hash FROM {attr} WHERE {attr} IN ({values})
|
||||
"""
|
||||
else:
|
||||
constraints[f'#_not_{attr}'] = f"""
|
||||
NOT EXISTS(
|
||||
SELECT 1 FROM {attr} WHERE
|
||||
{CLAIM_HASH_OR_REPOST_HASH_SQL}={attr}.claim_hash
|
||||
claim.claim_hash={attr}.claim_hash
|
||||
AND {attr} IN ({values})
|
||||
)
|
||||
"""
|
||||
@@ -52,12 +52,11 @@ class ZScore:
|
||||
return cls(), cls.step, cls.finalize
|
||||
|
||||
|
||||
def install(connection):
|
||||
def register_trending_functions(connection):
|
||||
connection.createaggregatefunction("zscore", ZScore.factory, 1)
|
||||
connection.cursor().execute(CREATE_TREND_TABLE)
|
||||
|
||||
|
||||
def run(db, height, final_height, affected_claims):
|
||||
def calculate_trending(db, height, final_height):
|
||||
# don't start tracking until we're at the end of initial sync
|
||||
if height < (final_height - (TRENDING_WINDOW * TRENDING_DATA_POINTS)):
|
||||
return
|
||||
@@ -1,9 +0,0 @@
|
||||
from . import zscore
|
||||
from . import ar
|
||||
from . import variable_decay
|
||||
|
||||
TRENDING_ALGORITHMS = {
|
||||
'zscore': zscore,
|
||||
'ar': ar,
|
||||
'variable_decay': variable_decay
|
||||
}
|
||||
@@ -1,265 +0,0 @@
|
||||
import copy
|
||||
import math
|
||||
import time
|
||||
|
||||
# Half life in blocks
|
||||
HALF_LIFE = 134
|
||||
|
||||
# Decay coefficient per block
|
||||
DECAY = 0.5**(1.0/HALF_LIFE)
|
||||
|
||||
# How frequently to write trending values to the db
|
||||
SAVE_INTERVAL = 10
|
||||
|
||||
# Renormalisation interval
|
||||
RENORM_INTERVAL = 1000
|
||||
|
||||
# Assertion
|
||||
assert RENORM_INTERVAL % SAVE_INTERVAL == 0
|
||||
|
||||
# Decay coefficient per renormalisation interval
|
||||
DECAY_PER_RENORM = DECAY**(RENORM_INTERVAL)
|
||||
|
||||
# Log trending calculations?
|
||||
TRENDING_LOG = True
|
||||
|
||||
|
||||
def install(connection):
|
||||
"""
|
||||
Install the AR trending algorithm.
|
||||
"""
|
||||
check_trending_values(connection)
|
||||
|
||||
if TRENDING_LOG:
|
||||
f = open("trending_ar.log", "a")
|
||||
f.close()
|
||||
|
||||
# Stub
|
||||
CREATE_TREND_TABLE = ""
|
||||
|
||||
|
||||
def check_trending_values(connection):
|
||||
"""
|
||||
If the trending values appear to be based on the zscore algorithm,
|
||||
reset them. This will allow resyncing from a standard snapshot.
|
||||
"""
|
||||
c = connection.cursor()
|
||||
needs_reset = False
|
||||
for row in c.execute("SELECT COUNT(*) num FROM claim WHERE trending_global <> 0;"):
|
||||
if row[0] != 0:
|
||||
needs_reset = True
|
||||
break
|
||||
|
||||
if needs_reset:
|
||||
print("Resetting some columns. This might take a while...", flush=True, end="")
|
||||
c.execute(""" BEGIN;
|
||||
UPDATE claim SET trending_group = 0;
|
||||
UPDATE claim SET trending_mixed = 0;
|
||||
UPDATE claim SET trending_global = 0;
|
||||
UPDATE claim SET trending_local = 0;
|
||||
COMMIT;""")
|
||||
print("done.")
|
||||
|
||||
|
||||
def spike_height(trending_score, x, x_old, time_boost=1.0):
|
||||
"""
|
||||
Compute the size of a trending spike.
|
||||
"""
|
||||
|
||||
# Change in softened amount
|
||||
change_in_softened_amount = x**0.25 - x_old**0.25
|
||||
|
||||
# Softened change in amount
|
||||
delta = x - x_old
|
||||
softened_change_in_amount = abs(delta)**0.25
|
||||
|
||||
# Softened change in amount counts more for minnows
|
||||
if delta > 0.0:
|
||||
if trending_score >= 0.0:
|
||||
multiplier = 0.1/((trending_score/time_boost + softened_change_in_amount) + 1.0)
|
||||
softened_change_in_amount *= multiplier
|
||||
else:
|
||||
softened_change_in_amount *= -1.0
|
||||
|
||||
return time_boost*(softened_change_in_amount + change_in_softened_amount)
|
||||
|
||||
|
||||
def get_time_boost(height):
|
||||
"""
|
||||
Return the time boost at a given height.
|
||||
"""
|
||||
return 1.0/DECAY**(height % RENORM_INTERVAL)
|
||||
|
||||
|
||||
def trending_log(s):
|
||||
"""
|
||||
Log a string.
|
||||
"""
|
||||
if TRENDING_LOG:
|
||||
fout = open("trending_ar.log", "a")
|
||||
fout.write(s)
|
||||
fout.flush()
|
||||
fout.close()
|
||||
|
||||
class TrendingData:
|
||||
"""
|
||||
An object of this class holds trending data
|
||||
"""
|
||||
def __init__(self):
|
||||
self.claims = {}
|
||||
|
||||
# Have all claims been read from db yet?
|
||||
self.initialised = False
|
||||
|
||||
def insert_claim_from_load(self, claim_hash, trending_score, total_amount):
|
||||
assert not self.initialised
|
||||
self.claims[claim_hash] = {"trending_score": trending_score,
|
||||
"total_amount": total_amount,
|
||||
"changed": False}
|
||||
|
||||
|
||||
def update_claim(self, claim_hash, total_amount, time_boost=1.0):
|
||||
"""
|
||||
Update trending data for a claim, given its new total amount.
|
||||
"""
|
||||
assert self.initialised
|
||||
|
||||
# Extract existing total amount and trending score
|
||||
# or use starting values if the claim is new
|
||||
if claim_hash in self.claims:
|
||||
old_state = copy.deepcopy(self.claims[claim_hash])
|
||||
else:
|
||||
old_state = {"trending_score": 0.0,
|
||||
"total_amount": 0.0,
|
||||
"changed": False}
|
||||
|
||||
# Calculate LBC change
|
||||
change = total_amount - old_state["total_amount"]
|
||||
|
||||
# Modify data if there was an LBC change
|
||||
if change != 0.0:
|
||||
spike = spike_height(old_state["trending_score"],
|
||||
total_amount,
|
||||
old_state["total_amount"],
|
||||
time_boost)
|
||||
trending_score = old_state["trending_score"] + spike
|
||||
self.claims[claim_hash] = {"total_amount": total_amount,
|
||||
"trending_score": trending_score,
|
||||
"changed": True}
|
||||
|
||||
|
||||
|
||||
def test_trending():
|
||||
"""
|
||||
Quick trending test for something receiving 10 LBC per block
|
||||
"""
|
||||
data = TrendingData()
|
||||
data.insert_claim_from_load("abc", 10.0, 1.0)
|
||||
data.initialised = True
|
||||
|
||||
for height in range(1, 5000):
|
||||
|
||||
if height % RENORM_INTERVAL == 0:
|
||||
data.claims["abc"]["trending_score"] *= DECAY_PER_RENORM
|
||||
|
||||
time_boost = get_time_boost(height)
|
||||
data.update_claim("abc", data.claims["abc"]["total_amount"] + 10.0,
|
||||
time_boost=time_boost)
|
||||
|
||||
|
||||
print(str(height) + " " + str(time_boost) + " " \
|
||||
+ str(data.claims["abc"]["trending_score"]))
|
||||
|
||||
|
||||
|
||||
# One global instance
|
||||
# pylint: disable=C0103
|
||||
trending_data = TrendingData()
|
||||
|
||||
def run(db, height, final_height, recalculate_claim_hashes):
|
||||
|
||||
if height < final_height - 5*HALF_LIFE:
|
||||
trending_log("Skipping AR trending at block {h}.\n".format(h=height))
|
||||
return
|
||||
|
||||
start = time.time()
|
||||
|
||||
trending_log("Calculating AR trending at block {h}.\n".format(h=height))
|
||||
trending_log(" Length of trending data = {l}.\n"\
|
||||
.format(l=len(trending_data.claims)))
|
||||
|
||||
# Renormalise trending scores and mark all as having changed
|
||||
if height % RENORM_INTERVAL == 0:
|
||||
trending_log(" Renormalising trending scores...")
|
||||
|
||||
keys = trending_data.claims.keys()
|
||||
for key in keys:
|
||||
if trending_data.claims[key]["trending_score"] != 0.0:
|
||||
trending_data.claims[key]["trending_score"] *= DECAY_PER_RENORM
|
||||
trending_data.claims[key]["changed"] = True
|
||||
|
||||
# Tiny becomes zero
|
||||
if abs(trending_data.claims[key]["trending_score"]) < 1E-9:
|
||||
trending_data.claims[key]["trending_score"] = 0.0
|
||||
|
||||
trending_log("done.\n")
|
||||
|
||||
|
||||
# Regular message.
|
||||
trending_log(" Reading total_amounts from db and updating"\
|
||||
+ " trending scores in RAM...")
|
||||
|
||||
# Get the value of the time boost
|
||||
time_boost = get_time_boost(height)
|
||||
|
||||
# Update claims from db
|
||||
if not trending_data.initialised:
|
||||
# On fresh launch
|
||||
for row in db.execute("""
|
||||
SELECT claim_hash, trending_mixed,
|
||||
(amount + support_amount)
|
||||
AS total_amount
|
||||
FROM claim;
|
||||
"""):
|
||||
trending_data.insert_claim_from_load(row[0], row[1], 1E-8*row[2])
|
||||
trending_data.initialised = True
|
||||
else:
|
||||
for row in db.execute(f"""
|
||||
SELECT claim_hash,
|
||||
(amount + support_amount)
|
||||
AS total_amount
|
||||
FROM claim
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in recalculate_claim_hashes)});
|
||||
""", recalculate_claim_hashes):
|
||||
trending_data.update_claim(row[0], 1E-8*row[1], time_boost)
|
||||
|
||||
trending_log("done.\n")
|
||||
|
||||
|
||||
# Write trending scores to DB
|
||||
if height % SAVE_INTERVAL == 0:
|
||||
|
||||
trending_log(" Writing trending scores to db...")
|
||||
|
||||
the_list = []
|
||||
keys = trending_data.claims.keys()
|
||||
for key in keys:
|
||||
if trending_data.claims[key]["changed"]:
|
||||
the_list.append((trending_data.claims[key]["trending_score"],
|
||||
key))
|
||||
trending_data.claims[key]["changed"] = False
|
||||
|
||||
trending_log("{n} scores to write...".format(n=len(the_list)))
|
||||
|
||||
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;",
|
||||
the_list)
|
||||
|
||||
trending_log("done.\n")
|
||||
|
||||
trending_log("Trending operations took {time} seconds.\n\n"\
|
||||
.format(time=time.time() - start))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_trending()
|
||||
@@ -1,485 +0,0 @@
|
||||
"""
|
||||
AR-like trending with a delayed effect and a faster
|
||||
decay rate for high valued claims.
|
||||
"""
|
||||
|
||||
import math
|
||||
import time
|
||||
import apsw
|
||||
|
||||
# Half life in blocks *for lower LBC claims* (it's shorter for whale claims)
|
||||
HALF_LIFE = 200
|
||||
|
||||
# Whale threshold, in LBC (higher -> less DB writing)
|
||||
WHALE_THRESHOLD = 10000.0
|
||||
|
||||
# Decay coefficient per block
|
||||
DECAY = 0.5**(1.0/HALF_LIFE)
|
||||
|
||||
# How frequently to write trending values to the db
|
||||
SAVE_INTERVAL = 10
|
||||
|
||||
# Renormalisation interval
|
||||
RENORM_INTERVAL = 1000
|
||||
|
||||
# Assertion
|
||||
assert RENORM_INTERVAL % SAVE_INTERVAL == 0
|
||||
|
||||
# Decay coefficient per renormalisation interval
|
||||
DECAY_PER_RENORM = DECAY**(RENORM_INTERVAL)
|
||||
|
||||
# Log trending calculations?
|
||||
TRENDING_LOG = True
|
||||
|
||||
|
||||
def install(connection):
|
||||
"""
|
||||
Install the trending algorithm.
|
||||
"""
|
||||
check_trending_values(connection)
|
||||
trending_data.initialise(connection.cursor())
|
||||
|
||||
if TRENDING_LOG:
|
||||
f = open("trending_variable_decay.log", "a")
|
||||
f.close()
|
||||
|
||||
# Stub
|
||||
CREATE_TREND_TABLE = ""
|
||||
|
||||
def check_trending_values(connection):
|
||||
"""
|
||||
If the trending values appear to be based on the zscore algorithm,
|
||||
reset them. This will allow resyncing from a standard snapshot.
|
||||
"""
|
||||
c = connection.cursor()
|
||||
needs_reset = False
|
||||
for row in c.execute("SELECT COUNT(*) num FROM claim WHERE trending_global <> 0;"):
|
||||
if row[0] != 0:
|
||||
needs_reset = True
|
||||
break
|
||||
|
||||
if needs_reset:
|
||||
print("Resetting some columns. This might take a while...", flush=True,
|
||||
end="")
|
||||
c.execute(""" BEGIN;
|
||||
UPDATE claim SET trending_group = 0;
|
||||
UPDATE claim SET trending_mixed = 0;
|
||||
COMMIT;""")
|
||||
print("done.")
|
||||
|
||||
|
||||
|
||||
|
||||
def trending_log(s):
|
||||
"""
|
||||
Log a string to the log file
|
||||
"""
|
||||
if TRENDING_LOG:
|
||||
fout = open("trending_variable_decay.log", "a")
|
||||
fout.write(s)
|
||||
fout.flush()
|
||||
fout.close()
|
||||
|
||||
|
||||
def trending_unit(height):
|
||||
"""
|
||||
Return the trending score unit at a given height.
|
||||
"""
|
||||
# Round to the beginning of a SAVE_INTERVAL batch of blocks.
|
||||
_height = height - (height % SAVE_INTERVAL)
|
||||
return 1.0/DECAY**(height % RENORM_INTERVAL)
|
||||
|
||||
|
||||
class TrendingDB:
|
||||
"""
|
||||
An in-memory database of trending scores
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.conn = apsw.Connection(":memory:")
|
||||
self.cursor = self.conn.cursor()
|
||||
self.initialised = False
|
||||
self.write_needed = set()
|
||||
|
||||
def execute(self, query, *args, **kwargs):
|
||||
return self.cursor.execute(query, *args, **kwargs)
|
||||
|
||||
def executemany(self, query, *args, **kwargs):
|
||||
return self.cursor.executemany(query, *args, **kwargs)
|
||||
|
||||
def begin(self):
|
||||
self.execute("BEGIN;")
|
||||
|
||||
def commit(self):
|
||||
self.execute("COMMIT;")
|
||||
|
||||
def initialise(self, db):
|
||||
"""
|
||||
Pass in claims.db
|
||||
"""
|
||||
if self.initialised:
|
||||
return
|
||||
|
||||
trending_log("Initialising trending database...")
|
||||
|
||||
# The need for speed
|
||||
self.execute("PRAGMA JOURNAL_MODE=OFF;")
|
||||
self.execute("PRAGMA SYNCHRONOUS=0;")
|
||||
|
||||
self.begin()
|
||||
|
||||
# Create the tables
|
||||
self.execute("""
|
||||
CREATE TABLE IF NOT EXISTS claims
|
||||
(claim_hash BYTES PRIMARY KEY,
|
||||
lbc REAL NOT NULL DEFAULT 0.0,
|
||||
trending_score REAL NOT NULL DEFAULT 0.0)
|
||||
WITHOUT ROWID;""")
|
||||
|
||||
self.execute("""
|
||||
CREATE TABLE IF NOT EXISTS spikes
|
||||
(id INTEGER PRIMARY KEY,
|
||||
claim_hash BYTES NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
mass REAL NOT NULL,
|
||||
FOREIGN KEY (claim_hash)
|
||||
REFERENCES claims (claim_hash));""")
|
||||
|
||||
# Clear out any existing data
|
||||
self.execute("DELETE FROM claims;")
|
||||
self.execute("DELETE FROM spikes;")
|
||||
|
||||
# Create indexes
|
||||
self.execute("CREATE INDEX idx1 ON spikes (claim_hash, height, mass);")
|
||||
self.execute("CREATE INDEX idx2 ON spikes (claim_hash, height, mass DESC);")
|
||||
self.execute("CREATE INDEX idx3 on claims (lbc DESC, claim_hash, trending_score);")
|
||||
|
||||
# Import data from claims.db
|
||||
for row in db.execute("""
|
||||
SELECT claim_hash,
|
||||
1E-8*(amount + support_amount) AS lbc,
|
||||
trending_mixed
|
||||
FROM claim;
|
||||
"""):
|
||||
self.execute("INSERT INTO claims VALUES (?, ?, ?);", row)
|
||||
self.commit()
|
||||
|
||||
self.initialised = True
|
||||
trending_log("done.\n")
|
||||
|
||||
def apply_spikes(self, height):
|
||||
"""
|
||||
Apply spikes that are due. This occurs inside a transaction.
|
||||
"""
|
||||
|
||||
spikes = []
|
||||
unit = trending_unit(height)
|
||||
for row in self.execute("""
|
||||
SELECT SUM(mass), claim_hash FROM spikes
|
||||
WHERE height = ?
|
||||
GROUP BY claim_hash;
|
||||
""", (height, )):
|
||||
spikes.append((row[0]*unit, row[1]))
|
||||
self.write_needed.add(row[1])
|
||||
|
||||
self.executemany("""
|
||||
UPDATE claims
|
||||
SET trending_score = (trending_score + ?)
|
||||
WHERE claim_hash = ?;
|
||||
""", spikes)
|
||||
self.execute("DELETE FROM spikes WHERE height = ?;", (height, ))
|
||||
|
||||
|
||||
def decay_whales(self, height):
|
||||
"""
|
||||
Occurs inside transaction.
|
||||
"""
|
||||
if height % SAVE_INTERVAL != 0:
|
||||
return
|
||||
|
||||
whales = self.execute("""
|
||||
SELECT trending_score, lbc, claim_hash
|
||||
FROM claims
|
||||
WHERE lbc >= ?;
|
||||
""", (WHALE_THRESHOLD, )).fetchall()
|
||||
whales2 = []
|
||||
for whale in whales:
|
||||
trending, lbc, claim_hash = whale
|
||||
|
||||
# Overall multiplication factor for decay rate
|
||||
# At WHALE_THRESHOLD, this is 1
|
||||
# At 10*WHALE_THRESHOLD, it is 3
|
||||
decay_rate_factor = 1.0 + 2.0*math.log10(lbc/WHALE_THRESHOLD)
|
||||
|
||||
# The -1 is because this is just the *extra* part being applied
|
||||
factor = (DECAY**SAVE_INTERVAL)**(decay_rate_factor - 1.0)
|
||||
|
||||
# Decay
|
||||
trending *= factor
|
||||
whales2.append((trending, claim_hash))
|
||||
self.write_needed.add(claim_hash)
|
||||
|
||||
self.executemany("UPDATE claims SET trending_score=? WHERE claim_hash=?;",
|
||||
whales2)
|
||||
|
||||
|
||||
def renorm(self, height):
|
||||
"""
|
||||
Renormalise trending scores. Occurs inside a transaction.
|
||||
"""
|
||||
|
||||
if height % RENORM_INTERVAL == 0:
|
||||
threshold = 1.0E-3/DECAY_PER_RENORM
|
||||
for row in self.execute("""SELECT claim_hash FROM claims
|
||||
WHERE ABS(trending_score) >= ?;""",
|
||||
(threshold, )):
|
||||
self.write_needed.add(row[0])
|
||||
|
||||
self.execute("""UPDATE claims SET trending_score = ?*trending_score
|
||||
WHERE ABS(trending_score) >= ?;""",
|
||||
(DECAY_PER_RENORM, threshold))
|
||||
|
||||
def write_to_claims_db(self, db, height):
|
||||
"""
|
||||
Write changed trending scores to claims.db.
|
||||
"""
|
||||
if height % SAVE_INTERVAL != 0:
|
||||
return
|
||||
|
||||
rows = self.execute(f"""
|
||||
SELECT trending_score, claim_hash
|
||||
FROM claims
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in self.write_needed)});
|
||||
""", self.write_needed).fetchall()
|
||||
|
||||
db.executemany("""UPDATE claim SET trending_mixed = ?
|
||||
WHERE claim_hash = ?;""", rows)
|
||||
|
||||
# Clear list of claims needing to be written to claims.db
|
||||
self.write_needed = set()
|
||||
|
||||
|
||||
def update(self, db, height, recalculate_claim_hashes):
|
||||
"""
|
||||
Update trending scores.
|
||||
Input is a cursor to claims.db, the block height, and the list of
|
||||
claims that changed.
|
||||
"""
|
||||
assert self.initialised
|
||||
|
||||
self.begin()
|
||||
self.renorm(height)
|
||||
|
||||
# Fetch changed/new claims from claims.db
|
||||
for row in db.execute(f"""
|
||||
SELECT claim_hash,
|
||||
1E-8*(amount + support_amount) AS lbc
|
||||
FROM claim
|
||||
WHERE claim_hash IN
|
||||
({','.join('?' for _ in recalculate_claim_hashes)});
|
||||
""", recalculate_claim_hashes):
|
||||
claim_hash, lbc = row
|
||||
|
||||
# Insert into trending db if it does not exist
|
||||
self.execute("""
|
||||
INSERT INTO claims (claim_hash)
|
||||
VALUES (?)
|
||||
ON CONFLICT (claim_hash) DO NOTHING;""",
|
||||
(claim_hash, ))
|
||||
|
||||
# See if it was an LBC change
|
||||
old = self.execute("SELECT * FROM claims WHERE claim_hash=?;",
|
||||
(claim_hash, )).fetchone()
|
||||
lbc_old = old[1]
|
||||
|
||||
# Save new LBC value into trending db
|
||||
self.execute("UPDATE claims SET lbc = ? WHERE claim_hash = ?;",
|
||||
(lbc, claim_hash))
|
||||
|
||||
if lbc > lbc_old:
|
||||
|
||||
# Schedule a future spike
|
||||
delay = min(int((lbc + 1E-8)**0.4), HALF_LIFE)
|
||||
spike = (claim_hash, height + delay, spike_mass(lbc, lbc_old))
|
||||
self.execute("""INSERT INTO spikes
|
||||
(claim_hash, height, mass)
|
||||
VALUES (?, ?, ?);""", spike)
|
||||
|
||||
elif lbc < lbc_old:
|
||||
|
||||
# Subtract from future spikes
|
||||
penalty = spike_mass(lbc_old, lbc)
|
||||
spikes = self.execute("""
|
||||
SELECT * FROM spikes
|
||||
WHERE claim_hash = ?
|
||||
ORDER BY height ASC, mass DESC;
|
||||
""", (claim_hash, )).fetchall()
|
||||
for spike in spikes:
|
||||
spike_id, mass = spike[0], spike[3]
|
||||
|
||||
if mass > penalty:
|
||||
# The entire penalty merely reduces this spike
|
||||
self.execute("UPDATE spikes SET mass=? WHERE id=?;",
|
||||
(mass - penalty, spike_id))
|
||||
penalty = 0.0
|
||||
else:
|
||||
# Removing this spike entirely accounts for some (or
|
||||
# all) of the penalty, then move on to other spikes
|
||||
self.execute("DELETE FROM spikes WHERE id=?;",
|
||||
(spike_id, ))
|
||||
penalty -= mass
|
||||
|
||||
# If penalty remains, that's a negative spike to be applied
|
||||
# immediately.
|
||||
if penalty > 0.0:
|
||||
self.execute("""
|
||||
INSERT INTO spikes (claim_hash, height, mass)
|
||||
VALUES (?, ?, ?);""",
|
||||
(claim_hash, height, -penalty))
|
||||
|
||||
self.apply_spikes(height)
|
||||
self.decay_whales(height)
|
||||
self.commit()
|
||||
|
||||
self.write_to_claims_db(db, height)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# The "global" instance to work with
|
||||
# pylint: disable=C0103
|
||||
trending_data = TrendingDB()
|
||||
|
||||
def spike_mass(x, x_old):
|
||||
"""
|
||||
Compute the mass of a trending spike (normed - constant units).
|
||||
x_old = old LBC value
|
||||
x = new LBC value
|
||||
"""
|
||||
|
||||
# Sign of trending spike
|
||||
sign = 1.0
|
||||
if x < x_old:
|
||||
sign = -1.0
|
||||
|
||||
# Magnitude
|
||||
mag = abs(x**0.25 - x_old**0.25)
|
||||
|
||||
# Minnow boost
|
||||
mag *= 1.0 + 2E4/(x + 100.0)**2
|
||||
|
||||
return sign*mag
|
||||
|
||||
|
||||
def run(db, height, final_height, recalculate_claim_hashes):
|
||||
if height < final_height - 5*HALF_LIFE:
|
||||
trending_log(f"Skipping trending calculations at block {height}.\n")
|
||||
return
|
||||
|
||||
start = time.time()
|
||||
trending_log(f"Calculating variable_decay trending at block {height}.\n")
|
||||
trending_data.update(db, height, recalculate_claim_hashes)
|
||||
end = time.time()
|
||||
trending_log(f"Trending operations took {end - start} seconds.\n\n")
|
||||
|
||||
def test_trending():
|
||||
"""
|
||||
Quick trending test for claims with different support patterns.
|
||||
Actually use the run() function.
|
||||
"""
|
||||
|
||||
# Create a fake "claims.db" for testing
|
||||
# pylint: disable=I1101
|
||||
dbc = apsw.Connection(":memory:")
|
||||
db = dbc.cursor()
|
||||
|
||||
# Create table
|
||||
db.execute("""
|
||||
BEGIN;
|
||||
CREATE TABLE claim (claim_hash TEXT PRIMARY KEY,
|
||||
amount REAL NOT NULL DEFAULT 0.0,
|
||||
support_amount REAL NOT NULL DEFAULT 0.0,
|
||||
trending_mixed REAL NOT NULL DEFAULT 0.0);
|
||||
COMMIT;
|
||||
""")
|
||||
|
||||
# Initialise trending data before anything happens with the claims
|
||||
trending_data.initialise(db)
|
||||
|
||||
# Insert initial states of claims
|
||||
everything = {"huge_whale": 0.01, "medium_whale": 0.01, "small_whale": 0.01,
|
||||
"huge_whale_botted": 0.01, "minnow": 0.01}
|
||||
|
||||
def to_list_of_tuples(stuff):
|
||||
l = []
|
||||
for key in stuff:
|
||||
l.append((key, stuff[key]))
|
||||
return l
|
||||
|
||||
db.executemany("""
|
||||
INSERT INTO claim (claim_hash, amount) VALUES (?, 1E8*?);
|
||||
""", to_list_of_tuples(everything))
|
||||
|
||||
# Process block zero
|
||||
height = 0
|
||||
run(db, height, height, everything.keys())
|
||||
|
||||
# Save trajectories for plotting
|
||||
trajectories = {}
|
||||
for row in trending_data.execute("""
|
||||
SELECT claim_hash, trending_score
|
||||
FROM claims;
|
||||
"""):
|
||||
trajectories[row[0]] = [row[1]/trending_unit(height)]
|
||||
|
||||
# Main loop
|
||||
for height in range(1, 1000):
|
||||
|
||||
# One-off supports
|
||||
if height == 1:
|
||||
everything["huge_whale"] += 5E5
|
||||
everything["medium_whale"] += 5E4
|
||||
everything["small_whale"] += 5E3
|
||||
|
||||
# Every block
|
||||
if height < 500:
|
||||
everything["huge_whale_botted"] += 5E5/500
|
||||
everything["minnow"] += 1
|
||||
|
||||
# Remove supports
|
||||
if height == 500:
|
||||
for key in everything:
|
||||
everything[key] = 0.01
|
||||
|
||||
# Whack into the db
|
||||
db.executemany("""
|
||||
UPDATE claim SET amount = 1E8*? WHERE claim_hash = ?;
|
||||
""", [(y, x) for (x, y) in to_list_of_tuples(everything)])
|
||||
|
||||
# Call run()
|
||||
run(db, height, height, everything.keys())
|
||||
|
||||
# Append current trending scores to trajectories
|
||||
for row in db.execute("""
|
||||
SELECT claim_hash, trending_mixed
|
||||
FROM claim;
|
||||
"""):
|
||||
trajectories[row[0]].append(row[1]/trending_unit(height))
|
||||
|
||||
dbc.close()
|
||||
|
||||
# pylint: disable=C0415
|
||||
import matplotlib.pyplot as plt
|
||||
for key in trajectories:
|
||||
plt.plot(trajectories[key], label=key)
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_trending()
|
||||
+73
-200
@@ -1,11 +1,12 @@
|
||||
import os
|
||||
|
||||
import apsw
|
||||
from typing import Union, Tuple, Set, List
|
||||
from itertools import chain
|
||||
from decimal import Decimal
|
||||
from collections import namedtuple
|
||||
from binascii import unhexlify, hexlify
|
||||
from multiprocessing import Manager
|
||||
from binascii import unhexlify
|
||||
|
||||
from lbry.wallet.server.leveldb import LevelDB
|
||||
from lbry.wallet.server.util import class_logger
|
||||
from lbry.wallet.database import query, constraints_to_sql
|
||||
@@ -15,10 +16,13 @@ from lbry.schema.mime_types import guess_stream_type
|
||||
from lbry.wallet import Ledger, RegTestLedger
|
||||
from lbry.wallet.transaction import Transaction, Output
|
||||
from lbry.wallet.server.db.canonical import register_canonical_functions
|
||||
from lbry.wallet.server.db.trending import TRENDING_ALGORITHMS
|
||||
from lbry.wallet.server.db.full_text_search import update_full_text_search, CREATE_FULL_TEXT_SEARCH, first_sync_finished
|
||||
from lbry.wallet.server.db.trending import (
|
||||
CREATE_TREND_TABLE, calculate_trending, register_trending_functions
|
||||
)
|
||||
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS
|
||||
|
||||
from .common import CLAIM_TYPES, STREAM_TYPES, COMMON_TAGS, INDEXED_LANGUAGES
|
||||
from lbry.wallet.server.db.elasticsearch import SearchIndex
|
||||
|
||||
ATTRIBUTE_ARRAY_MAX_LENGTH = 100
|
||||
|
||||
@@ -54,7 +58,6 @@ class SQLDB:
|
||||
description text,
|
||||
|
||||
claim_type integer,
|
||||
has_source bool,
|
||||
reposted integer default 0,
|
||||
|
||||
-- streams
|
||||
@@ -62,7 +65,6 @@ class SQLDB:
|
||||
media_type text,
|
||||
fee_amount integer default 0,
|
||||
fee_currency text,
|
||||
duration integer,
|
||||
|
||||
-- reposts
|
||||
reposted_claim_hash bytes,
|
||||
@@ -93,7 +95,6 @@ class SQLDB:
|
||||
create index if not exists claim_txo_hash_idx on claim (txo_hash);
|
||||
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
|
||||
create index if not exists claim_expiration_height_idx on claim (expiration_height);
|
||||
create index if not exists claim_reposted_claim_hash_idx on claim (reposted_claim_hash);
|
||||
"""
|
||||
|
||||
CREATE_SUPPORT_TABLE = """
|
||||
@@ -116,15 +117,6 @@ class SQLDB:
|
||||
create unique index if not exists tag_claim_hash_tag_idx on tag (claim_hash, tag);
|
||||
"""
|
||||
|
||||
CREATE_LANGUAGE_TABLE = """
|
||||
create table if not exists language (
|
||||
language text not null,
|
||||
claim_hash bytes not null,
|
||||
height integer not null
|
||||
);
|
||||
create unique index if not exists language_claim_hash_language_idx on language (claim_hash, language);
|
||||
"""
|
||||
|
||||
CREATE_CLAIMTRIE_TABLE = """
|
||||
create table if not exists claimtrie (
|
||||
normalized text primary key,
|
||||
@@ -134,62 +126,35 @@ class SQLDB:
|
||||
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
|
||||
"""
|
||||
|
||||
CREATE_CHANGELOG_TRIGGER = """
|
||||
create table if not exists changelog (
|
||||
claim_hash bytes primary key
|
||||
);
|
||||
create index if not exists claimtrie_claim_hash_idx on claimtrie (claim_hash);
|
||||
create trigger if not exists claim_changelog after update on claim
|
||||
begin
|
||||
insert or ignore into changelog (claim_hash) values (new.claim_hash);
|
||||
end;
|
||||
create trigger if not exists claimtrie_changelog after update on claimtrie
|
||||
begin
|
||||
insert or ignore into changelog (claim_hash) values (new.claim_hash);
|
||||
insert or ignore into changelog (claim_hash) values (old.claim_hash);
|
||||
end;
|
||||
"""
|
||||
|
||||
SEARCH_INDEXES = """
|
||||
-- used by any tag clouds
|
||||
create index if not exists tag_tag_idx on tag (tag, claim_hash);
|
||||
|
||||
-- naked order bys (no filters)
|
||||
create unique index if not exists claim_release_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_idx on claim (trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash);
|
||||
-- common ORDER BY
|
||||
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
|
||||
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
|
||||
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
|
||||
create unique index if not exists claim_trending_group_mixed_idx on claim (trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
|
||||
|
||||
-- claim_type filter + order by
|
||||
create unique index if not exists claim_type_release_idx on claim (release_time, claim_type, claim_hash);
|
||||
create unique index if not exists claim_type_trending_idx on claim (trending_group, trending_mixed, claim_type, claim_hash);
|
||||
create unique index if not exists claim_type_effective_amount_idx on claim (effective_amount, claim_type, claim_hash);
|
||||
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_global, trending_mixed, claim_hash);
|
||||
create unique index if not exists claim_type_release_idx on claim (claim_type, release_time, claim_hash);
|
||||
create unique index if not exists claim_type_effective_amount_idx on claim (claim_type, effective_amount, claim_hash);
|
||||
|
||||
-- stream_type filter + order by
|
||||
create unique index if not exists stream_type_release_idx on claim (stream_type, release_time, claim_hash);
|
||||
create unique index if not exists stream_type_trending_idx on claim (stream_type, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists stream_type_effective_amount_idx on claim (stream_type, effective_amount, claim_hash);
|
||||
|
||||
-- channel_hash filter + order by
|
||||
create unique index if not exists channel_hash_release_idx on claim (channel_hash, release_time, claim_hash);
|
||||
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists channel_hash_effective_amount_idx on claim (channel_hash, effective_amount, claim_hash);
|
||||
|
||||
-- duration filter + order by
|
||||
create unique index if not exists duration_release_idx on claim (duration, release_time, claim_hash);
|
||||
create unique index if not exists duration_trending_idx on claim (duration, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists duration_effective_amount_idx on claim (duration, effective_amount, claim_hash);
|
||||
|
||||
-- fee_amount + order by
|
||||
create unique index if not exists fee_amount_release_idx on claim (fee_amount, release_time, claim_hash);
|
||||
create unique index if not exists fee_amount_trending_idx on claim (fee_amount, trending_group, trending_mixed, claim_hash);
|
||||
create unique index if not exists fee_amount_effective_amount_idx on claim (fee_amount, effective_amount, claim_hash);
|
||||
create unique index if not exists channel_hash_release_time_idx on claim (channel_hash, release_time, claim_hash);
|
||||
|
||||
-- TODO: verify that all indexes below are used
|
||||
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
|
||||
|
||||
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
|
||||
|
||||
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
|
||||
create index if not exists claim_timestamp_idx on claim (timestamp);
|
||||
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
|
||||
|
||||
create index if not exists claim_stream_type_idx on claim (stream_type);
|
||||
create index if not exists claim_media_type_idx on claim (media_type);
|
||||
|
||||
create index if not exists claim_signature_valid_idx on claim (signature_valid);
|
||||
"""
|
||||
|
||||
@@ -198,27 +163,23 @@ class SQLDB:
|
||||
for tag_value, tag_key in COMMON_TAGS.items()
|
||||
)
|
||||
|
||||
LANGUAGE_INDEXES = '\n'.join(
|
||||
f"create unique index if not exists language_{language}_idx on language (language, claim_hash) WHERE language='{language}';"
|
||||
for language in INDEXED_LANGUAGES
|
||||
)
|
||||
|
||||
CREATE_TABLES_QUERY = (
|
||||
CREATE_CLAIM_TABLE +
|
||||
CREATE_TREND_TABLE +
|
||||
CREATE_FULL_TEXT_SEARCH +
|
||||
CREATE_SUPPORT_TABLE +
|
||||
CREATE_CLAIMTRIE_TABLE +
|
||||
CREATE_TAG_TABLE +
|
||||
CREATE_CHANGELOG_TRIGGER +
|
||||
CREATE_LANGUAGE_TABLE
|
||||
CREATE_TAG_TABLE
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self, main, path: str, blocking_channels: list, filtering_channels: list, trending: list):
|
||||
def __init__(self, main, path: str, blocking_channels: list, filtering_channels: list):
|
||||
self.main = main
|
||||
self._db_path = path
|
||||
self.db = None
|
||||
self.logger = class_logger(__name__, self.__class__.__name__)
|
||||
self.ledger = Ledger if main.coin.NET == 'mainnet' else RegTestLedger
|
||||
self._fts_synced = False
|
||||
self.state_manager = None
|
||||
self.blocked_streams = None
|
||||
self.blocked_channels = None
|
||||
self.blocking_channel_hashes = {
|
||||
@@ -229,8 +190,6 @@ class SQLDB:
|
||||
self.filtering_channel_hashes = {
|
||||
unhexlify(channel_id)[::-1] for channel_id in filtering_channels if channel_id
|
||||
}
|
||||
self.trending = trending
|
||||
self.pending_deletes = set()
|
||||
|
||||
def open(self):
|
||||
self.db = apsw.Connection(
|
||||
@@ -249,17 +208,19 @@ class SQLDB:
|
||||
self.execute(self.PRAGMAS)
|
||||
self.execute(self.CREATE_TABLES_QUERY)
|
||||
register_canonical_functions(self.db)
|
||||
self.blocked_streams = {}
|
||||
self.blocked_channels = {}
|
||||
self.filtered_streams = {}
|
||||
self.filtered_channels = {}
|
||||
register_trending_functions(self.db)
|
||||
self.state_manager = Manager()
|
||||
self.blocked_streams = self.state_manager.dict()
|
||||
self.blocked_channels = self.state_manager.dict()
|
||||
self.filtered_streams = self.state_manager.dict()
|
||||
self.filtered_channels = self.state_manager.dict()
|
||||
self.update_blocked_and_filtered_claims()
|
||||
for algorithm in self.trending:
|
||||
algorithm.install(self.db)
|
||||
|
||||
def close(self):
|
||||
if self.db is not None:
|
||||
self.db.close()
|
||||
if self.state_manager is not None:
|
||||
self.state_manager.shutdown()
|
||||
|
||||
def update_blocked_and_filtered_claims(self):
|
||||
self.update_claims_from_channel_hashes(
|
||||
@@ -275,10 +236,10 @@ class SQLDB:
|
||||
streams, channels = {}, {}
|
||||
if channel_hashes:
|
||||
sql = query(
|
||||
"SELECT repost.channel_hash, repost.reposted_claim_hash, target.claim_type "
|
||||
"FROM claim as repost JOIN claim AS target ON (target.claim_hash=repost.reposted_claim_hash)", **{
|
||||
'repost.reposted_claim_hash__is_not_null': 1,
|
||||
'repost.channel_hash__in': channel_hashes
|
||||
"SELECT claim.channel_hash, claim.reposted_claim_hash, reposted.claim_type "
|
||||
"FROM claim JOIN claim AS reposted ON (reposted.claim_hash=claim.reposted_claim_hash)", **{
|
||||
'claim.reposted_claim_hash__is_not_null': 1,
|
||||
'claim.channel_hash__in': channel_hashes
|
||||
}
|
||||
)
|
||||
for blocked_claim in self.execute(*sql):
|
||||
@@ -331,7 +292,7 @@ class SQLDB:
|
||||
self.execute('commit;')
|
||||
|
||||
def _upsertable_claims(self, txos: List[Output], header, clear_first=False):
|
||||
claim_hashes, claims, tags, languages = set(), [], {}, {}
|
||||
claim_hashes, claims, tags = set(), [], {}
|
||||
for txo in txos:
|
||||
tx = txo.tx_ref.tx
|
||||
|
||||
@@ -342,13 +303,6 @@ class SQLDB:
|
||||
#self.logger.exception(f"Could not decode claim name for {tx.id}:{txo.position}.")
|
||||
continue
|
||||
|
||||
language = 'none'
|
||||
try:
|
||||
if txo.claim.is_stream and txo.claim.stream.languages:
|
||||
language = txo.claim.stream.languages[0].language
|
||||
except:
|
||||
pass
|
||||
|
||||
claim_hash = txo.claim_hash
|
||||
claim_hashes.add(claim_hash)
|
||||
claim_record = {
|
||||
@@ -364,9 +318,7 @@ class SQLDB:
|
||||
'title': None,
|
||||
'description': None,
|
||||
'author': None,
|
||||
'duration': None,
|
||||
'claim_type': None,
|
||||
'has_source': False,
|
||||
'stream_type': None,
|
||||
'media_type': None,
|
||||
'release_time': None,
|
||||
@@ -384,16 +336,11 @@ class SQLDB:
|
||||
|
||||
if claim.is_stream:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['stream']
|
||||
claim_record['has_source'] = claim.stream.has_source
|
||||
claim_record['media_type'] = claim.stream.source.media_type
|
||||
claim_record['stream_type'] = STREAM_TYPES[guess_stream_type(claim_record['media_type'])]
|
||||
claim_record['title'] = claim.stream.title
|
||||
claim_record['description'] = claim.stream.description
|
||||
claim_record['author'] = claim.stream.author
|
||||
if claim.stream.video and claim.stream.video.duration:
|
||||
claim_record['duration'] = claim.stream.video.duration
|
||||
if claim.stream.audio and claim.stream.audio.duration:
|
||||
claim_record['duration'] = claim.stream.audio.duration
|
||||
if claim.stream.release_time:
|
||||
claim_record['release_time'] = claim.stream.release_time
|
||||
if claim.stream.has_fee:
|
||||
@@ -401,17 +348,11 @@ class SQLDB:
|
||||
if isinstance(fee.currency, str):
|
||||
claim_record['fee_currency'] = fee.currency.lower()
|
||||
if isinstance(fee.amount, Decimal):
|
||||
if fee.amount >= 0 and int(fee.amount*1000) < 9223372036854775807:
|
||||
claim_record['fee_amount'] = int(fee.amount*1000)
|
||||
claim_record['fee_amount'] = int(fee.amount*1000)
|
||||
elif claim.is_repost:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['repost']
|
||||
claim_record['reposted_claim_hash'] = claim.repost.reference.claim_hash
|
||||
elif claim.is_channel:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['channel']
|
||||
elif claim.is_collection:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['collection']
|
||||
|
||||
languages[(language, claim_hash)] = (language, claim_hash, tx.height)
|
||||
|
||||
for tag in clean_tags(claim.message.tags):
|
||||
tags[(tag, claim_hash)] = (tag, claim_hash, tx.height)
|
||||
@@ -423,10 +364,6 @@ class SQLDB:
|
||||
self.executemany(
|
||||
"INSERT OR IGNORE INTO tag (tag, claim_hash, height) VALUES (?, ?, ?)", tags.values()
|
||||
)
|
||||
if languages:
|
||||
self.executemany(
|
||||
"INSERT OR IGNORE INTO language (language, claim_hash, height) VALUES (?, ?, ?)", languages.values()
|
||||
)
|
||||
|
||||
return claims
|
||||
|
||||
@@ -434,15 +371,15 @@ class SQLDB:
|
||||
claims = self._upsertable_claims(txos, header)
|
||||
if claims:
|
||||
self.executemany("""
|
||||
INSERT OR REPLACE INTO claim (
|
||||
INSERT OR IGNORE INTO claim (
|
||||
claim_hash, claim_id, claim_name, normalized, txo_hash, tx_position, amount,
|
||||
claim_type, media_type, stream_type, timestamp, creation_timestamp, has_source,
|
||||
fee_currency, fee_amount, title, description, author, duration, height, reposted_claim_hash,
|
||||
claim_type, media_type, stream_type, timestamp, creation_timestamp,
|
||||
fee_currency, fee_amount, title, description, author, height, reposted_claim_hash,
|
||||
creation_height, release_time, activation_height, expiration_height, short_url)
|
||||
VALUES (
|
||||
:claim_hash, :claim_id, :claim_name, :normalized, :txo_hash, :tx_position, :amount,
|
||||
:claim_type, :media_type, :stream_type, :timestamp, :timestamp, :has_source,
|
||||
:fee_currency, :fee_amount, :title, :description, :author, :duration, :height, :reposted_claim_hash, :height,
|
||||
:claim_type, :media_type, :stream_type, :timestamp, :timestamp,
|
||||
:fee_currency, :fee_amount, :title, :description, :author, :height, :reposted_claim_hash, :height,
|
||||
CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE :timestamp END,
|
||||
CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END,
|
||||
CASE WHEN :height >= 137181 THEN :height+2102400 ELSE :height+262974 END,
|
||||
@@ -459,8 +396,8 @@ class SQLDB:
|
||||
UPDATE claim SET
|
||||
txo_hash=:txo_hash, tx_position=:tx_position, amount=:amount, height=:height,
|
||||
claim_type=:claim_type, media_type=:media_type, stream_type=:stream_type,
|
||||
timestamp=:timestamp, fee_amount=:fee_amount, fee_currency=:fee_currency, has_source=:has_source,
|
||||
title=:title, duration=:duration, description=:description, author=:author, reposted_claim_hash=:reposted_claim_hash,
|
||||
timestamp=:timestamp, fee_amount=:fee_amount, fee_currency=:fee_currency,
|
||||
title=:title, description=:description, author=:author, reposted_claim_hash=:reposted_claim_hash,
|
||||
release_time=CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE release_time END
|
||||
WHERE claim_hash=:claim_hash;
|
||||
""", claims)
|
||||
@@ -477,15 +414,6 @@ class SQLDB:
|
||||
return {r.channel_hash for r in affected_channels}
|
||||
return set()
|
||||
|
||||
def delete_claims_above_height(self, height: int):
|
||||
claim_hashes = [x[0] for x in self.execute(
|
||||
"SELECT claim_hash FROM claim WHERE height>?", (height, )
|
||||
).fetchall()]
|
||||
while claim_hashes:
|
||||
batch = set(claim_hashes[:500])
|
||||
claim_hashes = claim_hashes[500:]
|
||||
self.delete_claims(batch)
|
||||
|
||||
def _clear_claim_metadata(self, claim_hashes: Set[bytes]):
|
||||
if claim_hashes:
|
||||
for table in ('tag',): # 'language', 'location', etc
|
||||
@@ -543,7 +471,6 @@ class SQLDB:
|
||||
WHERE claim_hash = ?
|
||||
""", targets
|
||||
)
|
||||
return {target[0] for target in targets}
|
||||
|
||||
def validate_channel_signatures(self, height, new_claims, updated_claims, spent_claims, affected_channels, timer):
|
||||
if not new_claims and not updated_claims and not spent_claims:
|
||||
@@ -710,7 +637,7 @@ class SQLDB:
|
||||
claim_in_channel.channel_hash=claim.claim_hash
|
||||
)
|
||||
WHERE claim_hash = ?
|
||||
""", [(channel_hash,) for channel_hash in all_channel_keys])
|
||||
""", [(channel_hash,) for channel_hash in all_channel_keys.keys()])
|
||||
sub_timer.stop()
|
||||
|
||||
sub_timer = timer.add_timer('update blocked claims list')
|
||||
@@ -817,58 +744,11 @@ class SQLDB:
|
||||
f"SELECT claim_hash, normalized FROM claim WHERE expiration_height = {height}"
|
||||
)
|
||||
|
||||
def enqueue_changes(self):
|
||||
query = """
|
||||
SELECT claimtrie.claim_hash as is_controlling,
|
||||
claimtrie.last_take_over_height,
|
||||
(select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags,
|
||||
(select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages,
|
||||
(select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source,
|
||||
(select cr.claim_type from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_claim_type,
|
||||
claim.*
|
||||
FROM claim LEFT JOIN claimtrie USING (claim_hash)
|
||||
WHERE claim.claim_hash in (SELECT claim_hash FROM changelog)
|
||||
"""
|
||||
for claim in self.execute(query):
|
||||
claim = claim._asdict()
|
||||
id_set = set(filter(None, (claim['claim_hash'], claim['channel_hash'], claim['reposted_claim_hash'])))
|
||||
claim['censor_type'] = 0
|
||||
claim['censoring_channel_hash'] = None
|
||||
claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source'])
|
||||
for reason_id in id_set:
|
||||
if reason_id in self.blocked_streams:
|
||||
claim['censor_type'] = 2
|
||||
claim['censoring_channel_hash'] = self.blocked_streams.get(reason_id)
|
||||
elif reason_id in self.blocked_channels:
|
||||
claim['censor_type'] = 2
|
||||
claim['censoring_channel_hash'] = self.blocked_channels.get(reason_id)
|
||||
elif reason_id in self.filtered_streams:
|
||||
claim['censor_type'] = 1
|
||||
claim['censoring_channel_hash'] = self.filtered_streams.get(reason_id)
|
||||
elif reason_id in self.filtered_channels:
|
||||
claim['censor_type'] = 1
|
||||
claim['censoring_channel_hash'] = self.filtered_channels.get(reason_id)
|
||||
|
||||
claim['tags'] = claim['tags'].split(',,') if claim['tags'] else []
|
||||
claim['languages'] = claim['languages'].split(' ') if claim['languages'] else []
|
||||
yield 'update', claim
|
||||
|
||||
def clear_changelog(self):
|
||||
self.execute("delete from changelog;")
|
||||
|
||||
def claim_producer(self):
|
||||
while self.pending_deletes:
|
||||
claim_hash = self.pending_deletes.pop()
|
||||
yield 'delete', hexlify(claim_hash[::-1]).decode()
|
||||
for claim in self.enqueue_changes():
|
||||
yield claim
|
||||
self.clear_changelog()
|
||||
|
||||
def advance_txs(self, height, all_txs, header, daemon_height, timer):
|
||||
insert_claims = []
|
||||
update_claims = []
|
||||
update_claim_hashes = set()
|
||||
delete_claim_hashes = self.pending_deletes
|
||||
delete_claim_hashes = set()
|
||||
insert_supports = []
|
||||
delete_support_txo_hashes = set()
|
||||
recalculate_claim_hashes = set() # added/deleted supports, added/updated claim
|
||||
@@ -937,17 +817,27 @@ class SQLDB:
|
||||
expire_timer.stop()
|
||||
|
||||
r = timer.run
|
||||
r(update_full_text_search, 'before-delete',
|
||||
delete_claim_hashes, self.db.cursor(), self.main.first_sync)
|
||||
affected_channels = r(self.delete_claims, delete_claim_hashes)
|
||||
r(self.delete_supports, delete_support_txo_hashes)
|
||||
r(self.insert_claims, insert_claims, header)
|
||||
r(self.calculate_reposts, insert_claims)
|
||||
r(update_full_text_search, 'after-insert',
|
||||
[txo.claim_hash for txo in insert_claims], self.db.cursor(), self.main.first_sync)
|
||||
r(update_full_text_search, 'before-update',
|
||||
[txo.claim_hash for txo in update_claims], self.db.cursor(), self.main.first_sync)
|
||||
r(self.update_claims, update_claims, header)
|
||||
r(update_full_text_search, 'after-update',
|
||||
[txo.claim_hash for txo in update_claims], self.db.cursor(), self.main.first_sync)
|
||||
r(self.validate_channel_signatures, height, insert_claims,
|
||||
update_claims, delete_claim_hashes, affected_channels, forward_timer=True)
|
||||
r(self.insert_supports, insert_supports)
|
||||
r(self.update_claimtrie, height, recalculate_claim_hashes, deleted_claim_names, forward_timer=True)
|
||||
for algorithm in self.trending:
|
||||
r(algorithm.run, self.db.cursor(), height, daemon_height, recalculate_claim_hashes)
|
||||
r(calculate_trending, self.db.cursor(), height, daemon_height)
|
||||
if not self._fts_synced and self.main.first_sync and height == daemon_height:
|
||||
r(first_sync_finished, self.db.cursor())
|
||||
self._fts_synced = True
|
||||
|
||||
|
||||
class LBRYLevelDB(LevelDB):
|
||||
@@ -955,34 +845,17 @@ class LBRYLevelDB(LevelDB):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
path = os.path.join(self.env.db_dir, 'claims.db')
|
||||
trending = []
|
||||
for algorithm_name in self.env.trending_algorithms:
|
||||
if algorithm_name in TRENDING_ALGORITHMS:
|
||||
trending.append(TRENDING_ALGORITHMS[algorithm_name])
|
||||
if self.env.es_mode == 'reader':
|
||||
self.logger.info('Index mode: reader')
|
||||
self.sql = None
|
||||
else:
|
||||
self.logger.info('Index mode: writer. Using SQLite db to sync ES')
|
||||
self.sql = SQLDB(
|
||||
self, path,
|
||||
self.env.default('BLOCKING_CHANNEL_IDS', '').split(' '),
|
||||
self.env.default('FILTERING_CHANNEL_IDS', '').split(' '),
|
||||
trending
|
||||
)
|
||||
|
||||
# Search index
|
||||
self.search_index = SearchIndex(
|
||||
self.env.es_index_prefix, self.env.database_query_timeout, self.env.elastic_host, self.env.elastic_port
|
||||
# space separated list of channel URIs used for filtering bad content
|
||||
self.sql = SQLDB(
|
||||
self, path,
|
||||
self.env.default('BLOCKING_CHANNEL_IDS', '').split(' '),
|
||||
self.env.default('FILTERING_CHANNEL_IDS', '').split(' '),
|
||||
)
|
||||
|
||||
def close(self):
|
||||
super().close()
|
||||
if self.sql:
|
||||
self.sql.close()
|
||||
self.sql.close()
|
||||
|
||||
async def _open_dbs(self, *args, **kwargs):
|
||||
await self.search_index.start()
|
||||
await super()._open_dbs(*args, **kwargs)
|
||||
if self.sql:
|
||||
self.sql.open()
|
||||
self.sql.open()
|
||||
@@ -33,15 +33,10 @@ class Env:
|
||||
self.allow_root = self.boolean('ALLOW_ROOT', False)
|
||||
self.host = self.default('HOST', 'localhost')
|
||||
self.rpc_host = self.default('RPC_HOST', 'localhost')
|
||||
self.elastic_host = self.default('ELASTIC_HOST', 'localhost')
|
||||
self.elastic_port = self.integer('ELASTIC_PORT', 9200)
|
||||
self.loop_policy = self.set_event_loop_policy()
|
||||
self.obsolete(['UTXO_MB', 'HIST_MB', 'NETWORK'])
|
||||
self.db_dir = self.required('DB_DIRECTORY')
|
||||
self.db_engine = self.default('DB_ENGINE', 'leveldb')
|
||||
self.trending_algorithms = [
|
||||
trending for trending in set(self.default('TRENDING_ALGORITHMS', 'zscore').split(' ')) if trending
|
||||
]
|
||||
self.max_query_workers = self.integer('MAX_QUERY_WORKERS', None)
|
||||
self.individual_tag_indexes = self.boolean('INDIVIDUAL_TAG_INDEXES', True)
|
||||
self.track_metrics = self.boolean('TRACK_METRICS', False)
|
||||
@@ -55,13 +50,10 @@ class Env:
|
||||
coin_name = self.required('COIN').strip()
|
||||
network = self.default('NET', 'mainnet').strip()
|
||||
self.coin = Coin.lookup_coin_class(coin_name, network)
|
||||
self.es_index_prefix = self.default('ES_INDEX_PREFIX', '')
|
||||
self.es_mode = self.default('ES_MODE', 'writer')
|
||||
self.cache_MB = self.integer('CACHE_MB', 1200)
|
||||
self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
|
||||
# Server stuff
|
||||
self.tcp_port = self.integer('TCP_PORT', None)
|
||||
self.udp_port = self.integer('UDP_PORT', self.tcp_port)
|
||||
self.ssl_port = self.integer('SSL_PORT', None)
|
||||
if self.ssl_port:
|
||||
self.ssl_certfile = self.required('SSL_CERTFILE')
|
||||
@@ -73,7 +65,6 @@ class Env:
|
||||
self.tor_banner_file = self.default('TOR_BANNER_FILE', self.banner_file)
|
||||
self.anon_logs = self.boolean('ANON_LOGS', False)
|
||||
self.log_sessions = self.integer('LOG_SESSIONS', 3600)
|
||||
self.allow_lan_udp = self.boolean('ALLOW_LAN_UDP', False)
|
||||
# Peer discovery
|
||||
self.peer_discovery = self.peer_discovery_enum()
|
||||
self.peer_announce = self.boolean('PEER_ANNOUNCE', True)
|
||||
@@ -81,18 +72,16 @@ class Env:
|
||||
self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost')
|
||||
self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None)
|
||||
# The electrum client takes the empty string as unspecified
|
||||
self.payment_address = self.default('PAYMENT_ADDRESS', '')
|
||||
self.donation_address = self.default('DONATION_ADDRESS', '')
|
||||
# Server limits to help prevent DoS
|
||||
self.max_send = self.integer('MAX_SEND', 1000000)
|
||||
self.max_receive = self.integer('MAX_RECEIVE', 1000000)
|
||||
self.max_subs = self.integer('MAX_SUBS', 250000)
|
||||
self.max_sessions = self.sane_max_sessions()
|
||||
self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000)
|
||||
self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
|
||||
self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
|
||||
self.description = self.default('DESCRIPTION', '')
|
||||
self.daily_fee = self.string_amount('DAILY_FEE', '0')
|
||||
self.daily_fee = self.integer('DAILY_FEE', 0)
|
||||
|
||||
# Identities
|
||||
clearnet_identity = self.clearnet_identity()
|
||||
@@ -100,7 +89,7 @@ class Env:
|
||||
self.identities = [identity
|
||||
for identity in (clearnet_identity, tor_identity)
|
||||
if identity is not None]
|
||||
self.database_query_timeout = float(self.integer('QUERY_TIMEOUT_MS', 3000)) / 1000.0
|
||||
self.database_query_timeout = float(self.integer('QUERY_TIMEOUT_MS', 250)) / 1000.0
|
||||
|
||||
@classmethod
|
||||
def default(cls, envvar, default):
|
||||
@@ -118,14 +107,6 @@ class Env:
|
||||
raise cls.Error(f'required envvar {envvar} not set')
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def string_amount(cls, envvar, default):
|
||||
value = environ.get(envvar, default)
|
||||
amount_pattern = re.compile("[0-9]{0,10}(\.[0-9]{1,8})?")
|
||||
if len(value) > 0 and not amount_pattern.fullmatch(value):
|
||||
raise cls.Error(f'{value} is not a valid amount for {envvar}')
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def integer(cls, envvar, default):
|
||||
value = environ.get(envvar)
|
||||
|
||||
@@ -67,17 +67,17 @@ def hash160(x):
|
||||
return ripemd160(sha256(x))
|
||||
|
||||
|
||||
def hash_to_hex_str(x: bytes) -> str:
|
||||
def hash_to_hex_str(x):
|
||||
"""Convert a big-endian binary hash to displayed hex string.
|
||||
|
||||
Display form of a binary hash is reversed and converted to hex.
|
||||
"""
|
||||
return x[::-1].hex()
|
||||
return bytes(reversed(x)).hex()
|
||||
|
||||
|
||||
def hex_str_to_hash(x: str) -> bytes:
|
||||
def hex_str_to_hash(x):
|
||||
"""Convert a displayed hex string to a binary hash."""
|
||||
return hex_to_bytes(x)[::-1]
|
||||
return bytes(reversed(hex_to_bytes(x)))
|
||||
|
||||
|
||||
class Base58Error(Exception):
|
||||
|
||||
@@ -173,20 +173,20 @@ class History:
|
||||
|
||||
self.logger.info(f'backing up removed {nremoves:,d} history entries')
|
||||
|
||||
# def get_txnums(self, hashX, limit=1000):
|
||||
# """Generator that returns an unpruned, sorted list of tx_nums in the
|
||||
# history of a hashX. Includes both spending and receiving
|
||||
# transactions. By default yields at most 1000 entries. Set
|
||||
# limit to None to get them all. """
|
||||
# limit = util.resolve_limit(limit)
|
||||
# for key, hist in self.db.iterator(prefix=hashX):
|
||||
# a = array.array('I')
|
||||
# a.frombytes(hist)
|
||||
# for tx_num in a:
|
||||
# if limit == 0:
|
||||
# return
|
||||
# yield tx_num
|
||||
# limit -= 1
|
||||
def get_txnums(self, hashX, limit=1000):
|
||||
"""Generator that returns an unpruned, sorted list of tx_nums in the
|
||||
history of a hashX. Includes both spending and receiving
|
||||
transactions. By default yields at most 1000 entries. Set
|
||||
limit to None to get them all. """
|
||||
limit = util.resolve_limit(limit)
|
||||
for key, hist in self.db.iterator(prefix=hashX):
|
||||
a = array.array('I')
|
||||
a.frombytes(hist)
|
||||
for tx_num in a:
|
||||
if limit == 0:
|
||||
return
|
||||
yield tx_num
|
||||
limit -= 1
|
||||
|
||||
#
|
||||
# History compaction
|
||||
|
||||
+125
-253
@@ -12,20 +12,17 @@
|
||||
import asyncio
|
||||
import array
|
||||
import ast
|
||||
import base64
|
||||
import os
|
||||
import time
|
||||
import zlib
|
||||
import typing
|
||||
from typing import Optional, List, Tuple, Iterable
|
||||
from asyncio import sleep
|
||||
from bisect import bisect_right
|
||||
from collections import namedtuple
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
from struct import pack, unpack
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
|
||||
import attr
|
||||
from lbry.utils import LRUCacheWithMetrics
|
||||
|
||||
from lbry.wallet.server import util
|
||||
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
|
||||
from lbry.wallet.server.merkle import Merkle, MerkleCache
|
||||
@@ -35,13 +32,6 @@ from lbry.wallet.server.history import History
|
||||
|
||||
|
||||
UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value")
|
||||
HEADER_PREFIX = b'H'
|
||||
TX_COUNT_PREFIX = b'T'
|
||||
TX_HASH_PREFIX = b'X'
|
||||
TX_PREFIX = b'B'
|
||||
TX_NUM_PREFIX = b'N'
|
||||
BLOCK_HASH_PREFIX = b'C'
|
||||
|
||||
|
||||
|
||||
@attr.s(slots=True)
|
||||
@@ -49,8 +39,7 @@ class FlushData:
|
||||
height = attr.ib()
|
||||
tx_count = attr.ib()
|
||||
headers = attr.ib()
|
||||
block_hashes = attr.ib()
|
||||
block_txs = attr.ib()
|
||||
block_tx_hashes = attr.ib()
|
||||
# The following are flushed to the UTXO DB if undo_infos is not None
|
||||
undo_infos = attr.ib()
|
||||
adds = attr.ib()
|
||||
@@ -74,7 +63,14 @@ class LevelDB:
|
||||
self.logger = util.class_logger(__name__, self.__class__.__name__)
|
||||
self.env = env
|
||||
self.coin = env.coin
|
||||
self.executor = None
|
||||
|
||||
# Setup block header size handlers
|
||||
if self.coin.STATIC_BLOCK_HEADERS:
|
||||
self.header_offset = self.coin.static_header_offset
|
||||
self.header_len = self.coin.static_header_len
|
||||
else:
|
||||
self.header_offset = self.dynamic_header_offset
|
||||
self.header_len = self.dynamic_header_len
|
||||
|
||||
self.logger.info(f'switching current directory to {env.db_dir}')
|
||||
|
||||
@@ -82,8 +78,6 @@ class LevelDB:
|
||||
self.history = History()
|
||||
self.utxo_db = None
|
||||
self.tx_counts = None
|
||||
self.headers = None
|
||||
self.encoded_headers = LRUCacheWithMetrics(1 << 21, metric_name='encoded_headers', namespace='wallet_server')
|
||||
self.last_flush = time.time()
|
||||
|
||||
self.logger.info(f'using {self.env.db_engine} for DB backend')
|
||||
@@ -92,105 +86,59 @@ class LevelDB:
|
||||
self.merkle = Merkle()
|
||||
self.header_mc = MerkleCache(self.merkle, self.fs_block_hashes)
|
||||
|
||||
self.headers_db = None
|
||||
self.tx_db = None
|
||||
|
||||
self._tx_and_merkle_cache = LRUCacheWithMetrics(2 ** 17, metric_name='tx_and_merkle', namespace="wallet_server")
|
||||
self.total_transactions = None
|
||||
path = partial(os.path.join, self.env.db_dir)
|
||||
self.headers_file = util.LogicalFile(path('meta/headers'), 2, 16000000)
|
||||
self.tx_counts_file = util.LogicalFile(path('meta/txcounts'), 2, 2000000)
|
||||
self.hashes_file = util.LogicalFile(path('meta/hashes'), 4, 16000000)
|
||||
if not self.coin.STATIC_BLOCK_HEADERS:
|
||||
self.headers_offsets_file = util.LogicalFile(
|
||||
path('meta/headers_offsets'), 2, 16000000)
|
||||
|
||||
async def _read_tx_counts(self):
|
||||
if self.tx_counts is not None:
|
||||
return
|
||||
# tx_counts[N] has the cumulative number of txs at the end of
|
||||
# height N. So tx_counts[0] is 1 - the genesis coinbase
|
||||
|
||||
def get_counts():
|
||||
return tuple(
|
||||
util.unpack_be_uint64(tx_count)
|
||||
for tx_count in self.tx_db.iterator(prefix=TX_COUNT_PREFIX, include_key=False)
|
||||
)
|
||||
|
||||
tx_counts = await asyncio.get_event_loop().run_in_executor(self.executor, get_counts)
|
||||
assert len(tx_counts) == self.db_height + 1, f"{len(tx_counts)} vs {self.db_height + 1}"
|
||||
size = (self.db_height + 1) * 4
|
||||
tx_counts = self.tx_counts_file.read(0, size)
|
||||
assert len(tx_counts) == size
|
||||
self.tx_counts = array.array('I', tx_counts)
|
||||
|
||||
if self.tx_counts:
|
||||
assert self.db_tx_count == self.tx_counts[-1], \
|
||||
f"{self.db_tx_count} vs {self.tx_counts[-1]} ({len(self.tx_counts)} counts)"
|
||||
assert self.db_tx_count == self.tx_counts[-1]
|
||||
else:
|
||||
assert self.db_tx_count == 0
|
||||
|
||||
async def _read_txids(self):
|
||||
def get_txids():
|
||||
return list(self.tx_db.iterator(prefix=TX_HASH_PREFIX, include_key=False))
|
||||
|
||||
start = time.perf_counter()
|
||||
self.logger.info("loading txids")
|
||||
txids = await asyncio.get_event_loop().run_in_executor(self.executor, get_txids)
|
||||
assert len(txids) == len(self.tx_counts) == 0 or len(txids) == self.tx_counts[-1]
|
||||
self.total_transactions = txids
|
||||
ts = time.perf_counter() - start
|
||||
self.logger.info("loaded %i txids in %ss", len(self.total_transactions), round(ts, 4))
|
||||
|
||||
async def _read_headers(self):
|
||||
if self.headers is not None:
|
||||
return
|
||||
|
||||
def get_headers():
|
||||
return list(self.headers_db.iterator(prefix=HEADER_PREFIX, include_key=False))
|
||||
|
||||
headers = await asyncio.get_event_loop().run_in_executor(self.executor, get_headers)
|
||||
assert len(headers) - 1 == self.db_height, f"{len(headers)} vs {self.db_height}"
|
||||
self.headers = headers
|
||||
|
||||
async def _open_dbs(self, for_sync, compacting):
|
||||
if self.executor is None:
|
||||
self.executor = ThreadPoolExecutor(1)
|
||||
coin_path = os.path.join(self.env.db_dir, 'COIN')
|
||||
if not os.path.isfile(coin_path):
|
||||
with util.open_file(coin_path, create=True) as f:
|
||||
f.write(f'ElectrumX databases and metadata for '
|
||||
f'{self.coin.NAME} {self.coin.NET}'.encode())
|
||||
|
||||
assert self.headers_db is None
|
||||
self.headers_db = self.db_class('headers', for_sync)
|
||||
if self.headers_db.is_new:
|
||||
self.logger.info('created new headers db')
|
||||
self.logger.info(f'opened headers DB (for sync: {for_sync})')
|
||||
|
||||
assert self.tx_db is None
|
||||
self.tx_db = self.db_class('tx', for_sync)
|
||||
if self.tx_db.is_new:
|
||||
self.logger.info('created new tx db')
|
||||
self.logger.info(f'opened tx DB (for sync: {for_sync})')
|
||||
|
||||
assert self.utxo_db is None
|
||||
|
||||
# First UTXO DB
|
||||
self.utxo_db = self.db_class('utxo', for_sync)
|
||||
if self.utxo_db.is_new:
|
||||
self.logger.info('created new utxo db')
|
||||
self.logger.info(f'opened utxo db (for sync: {for_sync})')
|
||||
self.logger.info('created new database')
|
||||
self.logger.info('creating metadata directory')
|
||||
os.mkdir(os.path.join(self.env.db_dir, 'meta'))
|
||||
coin_path = os.path.join(self.env.db_dir, 'meta', 'COIN')
|
||||
with util.open_file(coin_path, create=True) as f:
|
||||
f.write(f'ElectrumX databases and metadata for '
|
||||
f'{self.coin.NAME} {self.coin.NET}'.encode())
|
||||
if not self.coin.STATIC_BLOCK_HEADERS:
|
||||
self.headers_offsets_file.write(0, bytes(8))
|
||||
else:
|
||||
self.logger.info(f'opened UTXO DB (for sync: {for_sync})')
|
||||
self.read_utxo_state()
|
||||
|
||||
# Then history DB
|
||||
self.utxo_flush_count = self.history.open_db(
|
||||
self.db_class, for_sync, self.utxo_flush_count, compacting
|
||||
)
|
||||
self.utxo_flush_count = self.history.open_db(self.db_class, for_sync,
|
||||
self.utxo_flush_count,
|
||||
compacting)
|
||||
self.clear_excess_undo_info()
|
||||
|
||||
# Read TX counts (requires meta directory)
|
||||
await self._read_tx_counts()
|
||||
if self.total_transactions is None:
|
||||
await self._read_txids()
|
||||
await self._read_headers()
|
||||
|
||||
def close(self):
|
||||
self.utxo_db.close()
|
||||
self.history.close_db()
|
||||
self.headers_db.close()
|
||||
self.tx_db.close()
|
||||
self.executor.shutdown(wait=True)
|
||||
self.executor = None
|
||||
|
||||
async def open_for_compacting(self):
|
||||
await self._open_dbs(True, True)
|
||||
@@ -202,28 +150,18 @@ class LevelDB:
|
||||
synchronization. When serving clients we want the open files for
|
||||
serving network connections.
|
||||
"""
|
||||
self.logger.info("opened for sync")
|
||||
await self._open_dbs(True, False)
|
||||
|
||||
async def open_for_serving(self):
|
||||
"""Open the databases for serving. If they are already open they are
|
||||
closed first.
|
||||
"""
|
||||
self.logger.info('closing DBs to re-open for serving')
|
||||
if self.utxo_db:
|
||||
self.logger.info('closing DBs to re-open for serving')
|
||||
self.utxo_db.close()
|
||||
self.history.close_db()
|
||||
self.utxo_db = None
|
||||
if self.headers_db:
|
||||
self.headers_db.close()
|
||||
self.headers_db = None
|
||||
if self.tx_db:
|
||||
self.tx_db.close()
|
||||
self.tx_db = None
|
||||
|
||||
await self._open_dbs(False, False)
|
||||
self.logger.info("opened for serving")
|
||||
|
||||
# Header merkle cache
|
||||
|
||||
@@ -245,7 +183,7 @@ class LevelDB:
|
||||
assert flush_data.height == self.fs_height == self.db_height
|
||||
assert flush_data.tip == self.db_tip
|
||||
assert not flush_data.headers
|
||||
assert not flush_data.block_txs
|
||||
assert not flush_data.block_tx_hashes
|
||||
assert not flush_data.adds
|
||||
assert not flush_data.deletes
|
||||
assert not flush_data.undo_infos
|
||||
@@ -303,50 +241,36 @@ class LevelDB:
|
||||
"""
|
||||
prior_tx_count = (self.tx_counts[self.fs_height]
|
||||
if self.fs_height >= 0 else 0)
|
||||
assert len(flush_data.block_txs) == len(flush_data.headers)
|
||||
assert len(flush_data.block_tx_hashes) == len(flush_data.headers)
|
||||
assert flush_data.height == self.fs_height + len(flush_data.headers)
|
||||
assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
|
||||
else 0)
|
||||
assert len(self.tx_counts) == flush_data.height + 1
|
||||
assert len(
|
||||
b''.join(hashes for hashes, _ in flush_data.block_txs)
|
||||
) // 32 == flush_data.tx_count - prior_tx_count
|
||||
hashes = b''.join(flush_data.block_tx_hashes)
|
||||
flush_data.block_tx_hashes.clear()
|
||||
assert len(hashes) % 32 == 0
|
||||
assert len(hashes) // 32 == flush_data.tx_count - prior_tx_count
|
||||
|
||||
# Write the headers
|
||||
start_time = time.perf_counter()
|
||||
|
||||
with self.headers_db.write_batch() as batch:
|
||||
batch_put = batch.put
|
||||
for i, header in enumerate(flush_data.headers):
|
||||
batch_put(HEADER_PREFIX + util.pack_be_uint64(self.fs_height + i + 1), header)
|
||||
self.headers.append(header)
|
||||
# Write the headers, tx counts, and tx hashes
|
||||
start_time = time.time()
|
||||
height_start = self.fs_height + 1
|
||||
offset = self.header_offset(height_start)
|
||||
self.headers_file.write(offset, b''.join(flush_data.headers))
|
||||
self.fs_update_header_offsets(offset, height_start, flush_data.headers)
|
||||
flush_data.headers.clear()
|
||||
|
||||
height_start = self.fs_height + 1
|
||||
tx_num = prior_tx_count
|
||||
|
||||
with self.tx_db.write_batch() as batch:
|
||||
batch_put = batch.put
|
||||
for block_hash, (tx_hashes, txs) in zip(flush_data.block_hashes, flush_data.block_txs):
|
||||
tx_count = self.tx_counts[height_start]
|
||||
batch_put(BLOCK_HASH_PREFIX + util.pack_be_uint64(height_start), block_hash[::-1])
|
||||
batch_put(TX_COUNT_PREFIX + util.pack_be_uint64(height_start), util.pack_be_uint64(tx_count))
|
||||
height_start += 1
|
||||
offset = 0
|
||||
while offset < len(tx_hashes):
|
||||
batch_put(TX_HASH_PREFIX + util.pack_be_uint64(tx_num), tx_hashes[offset:offset+32])
|
||||
batch_put(TX_NUM_PREFIX + tx_hashes[offset:offset+32], util.pack_be_uint64(tx_num))
|
||||
batch_put(TX_PREFIX + tx_hashes[offset:offset+32], txs[offset // 32])
|
||||
tx_num += 1
|
||||
offset += 32
|
||||
|
||||
flush_data.block_txs.clear()
|
||||
flush_data.block_hashes.clear()
|
||||
offset = height_start * self.tx_counts.itemsize
|
||||
self.tx_counts_file.write(offset,
|
||||
self.tx_counts[height_start:].tobytes())
|
||||
offset = prior_tx_count * 32
|
||||
self.hashes_file.write(offset, hashes)
|
||||
|
||||
self.fs_height = flush_data.height
|
||||
self.fs_tx_count = flush_data.tx_count
|
||||
elapsed = time.perf_counter() - start_time
|
||||
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
|
||||
|
||||
if self.utxo_db.for_sync:
|
||||
elapsed = time.time() - start_time
|
||||
self.logger.info(f'flushed filesystem data in {elapsed:.2f}s')
|
||||
|
||||
def flush_history(self):
|
||||
self.history.flush()
|
||||
@@ -405,7 +329,7 @@ class LevelDB:
|
||||
def flush_backup(self, flush_data, touched):
|
||||
"""Like flush_dbs() but when backing up. All UTXOs are flushed."""
|
||||
assert not flush_data.headers
|
||||
assert not flush_data.block_txs
|
||||
assert not flush_data.block_tx_hashes
|
||||
assert flush_data.height < self.db_height
|
||||
self.history.assert_flushed()
|
||||
|
||||
@@ -424,33 +348,43 @@ class LevelDB:
|
||||
f'{elapsed:.1f}s. Height {flush_data.height:,d} '
|
||||
f'txs: {flush_data.tx_count:,d} ({tx_delta:+,d})')
|
||||
|
||||
def fs_update_header_offsets(self, offset_start, height_start, headers):
|
||||
if self.coin.STATIC_BLOCK_HEADERS:
|
||||
return
|
||||
offset = offset_start
|
||||
offsets = []
|
||||
for h in headers:
|
||||
offset += len(h)
|
||||
offsets.append(pack("<Q", offset))
|
||||
# For each header we get the offset of the next header, hence we
|
||||
# start writing from the next height
|
||||
pos = (height_start + 1) * 8
|
||||
self.headers_offsets_file.write(pos, b''.join(offsets))
|
||||
|
||||
def dynamic_header_offset(self, height):
|
||||
assert not self.coin.STATIC_BLOCK_HEADERS
|
||||
offset, = unpack('<Q', self.headers_offsets_file.read(height * 8, 8))
|
||||
return offset
|
||||
|
||||
def dynamic_header_len(self, height):
|
||||
return self.dynamic_header_offset(height + 1)\
|
||||
- self.dynamic_header_offset(height)
|
||||
|
||||
def backup_fs(self, height, tx_count):
|
||||
"""Back up during a reorg. This just updates our pointers."""
|
||||
while self.fs_height > height:
|
||||
self.fs_height -= 1
|
||||
self.headers.pop()
|
||||
self.fs_height = height
|
||||
self.fs_tx_count = tx_count
|
||||
# Truncate header_mc: header count is 1 more than the height.
|
||||
self.header_mc.truncate(height + 1)
|
||||
|
||||
def raw_header(self, height):
|
||||
async def raw_header(self, height):
|
||||
"""Return the binary header at the given height."""
|
||||
header, n = self.read_headers(height, 1)
|
||||
header, n = await self.read_headers(height, 1)
|
||||
if n != 1:
|
||||
raise IndexError(f'height {height:,d} out of range')
|
||||
return header
|
||||
|
||||
def encode_headers(self, start_height, count, headers):
|
||||
key = (start_height, count)
|
||||
if not self.encoded_headers.get(key):
|
||||
compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9)
|
||||
headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode()
|
||||
if start_height % 1000 != 0:
|
||||
return headers
|
||||
self.encoded_headers[key] = headers
|
||||
return self.encoded_headers.get(key)
|
||||
|
||||
def read_headers(self, start_height, count) -> typing.Tuple[bytes, int]:
|
||||
async def read_headers(self, start_height, count):
|
||||
"""Requires start_height >= 0, count >= 0. Reads as many headers as
|
||||
are available starting at start_height up to count. This
|
||||
would be zero if start_height is beyond self.db_height, for
|
||||
@@ -459,15 +393,20 @@ class LevelDB:
|
||||
Returns a (binary, n) pair where binary is the concatenated
|
||||
binary headers, and n is the count of headers returned.
|
||||
"""
|
||||
|
||||
if start_height < 0 or count < 0:
|
||||
raise self.DBError(f'{count:,d} headers starting at '
|
||||
f'{start_height:,d} not on disk')
|
||||
|
||||
disk_count = max(0, min(count, self.db_height + 1 - start_height))
|
||||
if disk_count:
|
||||
return b''.join(self.headers[start_height:start_height + disk_count]), disk_count
|
||||
return b'', 0
|
||||
def read_headers():
|
||||
# Read some from disk
|
||||
disk_count = max(0, min(count, self.db_height + 1 - start_height))
|
||||
if disk_count:
|
||||
offset = self.header_offset(start_height)
|
||||
size = self.header_offset(start_height + disk_count) - offset
|
||||
return self.headers_file.read(offset, size), disk_count
|
||||
return b'', 0
|
||||
|
||||
return await asyncio.get_event_loop().run_in_executor(None, read_headers)
|
||||
|
||||
def fs_tx_hash(self, tx_num):
|
||||
"""Return a par (tx_hash, tx_height) for the given tx number.
|
||||
@@ -475,67 +414,23 @@ class LevelDB:
|
||||
If the tx_height is not on disk, returns (None, tx_height)."""
|
||||
tx_height = bisect_right(self.tx_counts, tx_num)
|
||||
if tx_height > self.db_height:
|
||||
return None, tx_height
|
||||
try:
|
||||
return self.total_transactions[tx_num], tx_height
|
||||
except IndexError:
|
||||
self.logger.exception(
|
||||
"Failed to access a cached transaction, known bug #3142 "
|
||||
"should be fixed in #3205"
|
||||
)
|
||||
return None, tx_height
|
||||
|
||||
def _fs_transactions(self, txids: Iterable[str]):
|
||||
unpack_be_uint64 = util.unpack_be_uint64
|
||||
tx_counts = self.tx_counts
|
||||
tx_db_get = self.tx_db.get
|
||||
tx_cache = self._tx_and_merkle_cache
|
||||
|
||||
tx_infos = {}
|
||||
|
||||
for tx_hash in txids:
|
||||
cached_tx = tx_cache.get(tx_hash)
|
||||
if cached_tx:
|
||||
tx, merkle = cached_tx
|
||||
else:
|
||||
tx_hash_bytes = bytes.fromhex(tx_hash)[::-1]
|
||||
tx_num = tx_db_get(TX_NUM_PREFIX + tx_hash_bytes)
|
||||
tx = None
|
||||
tx_height = -1
|
||||
if tx_num is not None:
|
||||
tx_num = unpack_be_uint64(tx_num)
|
||||
tx_height = bisect_right(tx_counts, tx_num)
|
||||
if tx_height < self.db_height:
|
||||
tx = tx_db_get(TX_PREFIX + tx_hash_bytes)
|
||||
if tx_height == -1:
|
||||
merkle = {
|
||||
'block_height': -1
|
||||
}
|
||||
else:
|
||||
tx_pos = tx_num - tx_counts[tx_height - 1]
|
||||
branch, root = self.merkle.branch_and_root(
|
||||
self.total_transactions[tx_counts[tx_height - 1]:tx_counts[tx_height]], tx_pos
|
||||
)
|
||||
merkle = {
|
||||
'block_height': tx_height,
|
||||
'merkle': [
|
||||
hash_to_hex_str(hash)
|
||||
for hash in branch
|
||||
],
|
||||
'pos': tx_pos
|
||||
}
|
||||
if tx_height + 10 < self.db_height:
|
||||
tx_cache[tx_hash] = tx, merkle
|
||||
tx_infos[tx_hash] = (None if not tx else tx.hex(), merkle)
|
||||
return tx_infos
|
||||
|
||||
async def fs_transactions(self, txids):
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, self._fs_transactions, txids)
|
||||
tx_hash = None
|
||||
else:
|
||||
tx_hash = self.hashes_file.read(tx_num * 32, 32)
|
||||
return tx_hash, tx_height
|
||||
|
||||
async def fs_block_hashes(self, height, count):
|
||||
if height + count > len(self.headers):
|
||||
raise self.DBError(f'only got {len(self.headers) - height:,d} headers starting at {height:,d}, not {count:,d}')
|
||||
return [self.coin.header_hash(header) for header in self.headers[height:height + count]]
|
||||
headers_concat, headers_count = await self.read_headers(height, count)
|
||||
if headers_count != count:
|
||||
raise self.DBError(f'only got {headers_count:,d} headers starting at {height:,d}, not {count:,d}')
|
||||
offset = 0
|
||||
headers = []
|
||||
for n in range(count):
|
||||
hlen = self.header_len(height + n)
|
||||
headers.append(headers_concat[offset:offset + hlen])
|
||||
offset += hlen
|
||||
|
||||
return [self.coin.header_hash(header) for header in headers]
|
||||
|
||||
async def limited_history(self, hashX, *, limit=1000):
|
||||
"""Return an unpruned, sorted list of (tx_hash, height) tuples of
|
||||
@@ -544,33 +439,15 @@ class LevelDB:
|
||||
transactions. By default returns at most 1000 entries. Set
|
||||
limit to None to get them all.
|
||||
"""
|
||||
|
||||
def read_history():
|
||||
db_height = self.db_height
|
||||
tx_counts = self.tx_counts
|
||||
|
||||
cnt = 0
|
||||
txs = []
|
||||
|
||||
for hist in self.history.db.iterator(prefix=hashX, include_key=False):
|
||||
a = array.array('I')
|
||||
a.frombytes(hist)
|
||||
for tx_num in a:
|
||||
tx_height = bisect_right(tx_counts, tx_num)
|
||||
if tx_height > db_height:
|
||||
return
|
||||
txs.append((tx_num, tx_height))
|
||||
cnt += 1
|
||||
if limit and cnt >= limit:
|
||||
break
|
||||
if limit and cnt >= limit:
|
||||
break
|
||||
return txs
|
||||
tx_nums = list(self.history.get_txnums(hashX, limit))
|
||||
fs_tx_hash = self.fs_tx_hash
|
||||
return [fs_tx_hash(tx_num) for tx_num in tx_nums]
|
||||
|
||||
while True:
|
||||
history = await asyncio.get_event_loop().run_in_executor(self.executor, read_history)
|
||||
if history is not None:
|
||||
return [(self.total_transactions[tx_num], tx_height) for (tx_num, tx_height) in history]
|
||||
history = await asyncio.get_event_loop().run_in_executor(None, read_history)
|
||||
if all(hash is not None for hash, height in history):
|
||||
return history
|
||||
self.logger.warning(f'limited_history: tx hash '
|
||||
f'not found (reorg?), retrying...')
|
||||
await sleep(0.25)
|
||||
@@ -595,20 +472,16 @@ class LevelDB:
|
||||
batch_put(self.undo_key(height), b''.join(undo_info))
|
||||
|
||||
def raw_block_prefix(self):
|
||||
return 'block'
|
||||
return 'meta/block'
|
||||
|
||||
def raw_block_path(self, height):
|
||||
return os.path.join(self.env.db_dir, f'{self.raw_block_prefix()}{height:d}')
|
||||
|
||||
async def read_raw_block(self, height):
|
||||
def read_raw_block(self, height):
|
||||
"""Returns a raw block read from disk. Raises FileNotFoundError
|
||||
if the block isn't on-disk."""
|
||||
|
||||
def read():
|
||||
with util.open_file(self.raw_block_path(height)) as f:
|
||||
return f.read(-1)
|
||||
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, read)
|
||||
with util.open_file(self.raw_block_path(height)) as f:
|
||||
return f.read(-1)
|
||||
|
||||
def write_raw_block(self, block, height):
|
||||
"""Write a raw block to disk."""
|
||||
@@ -727,19 +600,18 @@ class LevelDB:
|
||||
utxos = []
|
||||
utxos_append = utxos.append
|
||||
s_unpack = unpack
|
||||
fs_tx_hash = self.fs_tx_hash
|
||||
# Key: b'u' + address_hashX + tx_idx + tx_num
|
||||
# Value: the UTXO value as a 64-bit unsigned integer
|
||||
prefix = b'u' + hashX
|
||||
for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
|
||||
tx_pos, tx_num = s_unpack('<HI', db_key[-6:])
|
||||
value, = unpack('<Q', db_value)
|
||||
tx_hash, height = fs_tx_hash(tx_num)
|
||||
tx_hash, height = self.fs_tx_hash(tx_num)
|
||||
utxos_append(UTXO(tx_num, tx_pos, tx_hash, height, value))
|
||||
return utxos
|
||||
|
||||
while True:
|
||||
utxos = await asyncio.get_event_loop().run_in_executor(self.executor, read_utxos)
|
||||
utxos = await asyncio.get_event_loop().run_in_executor(None, read_utxos)
|
||||
if all(utxo.tx_hash is not None for utxo in utxos):
|
||||
return utxos
|
||||
self.logger.warning(f'all_utxos: tx hash not '
|
||||
@@ -792,5 +664,5 @@ class LevelDB:
|
||||
return hashX, value
|
||||
return [lookup_utxo(*hashX_pair) for hashX_pair in hashX_pairs]
|
||||
|
||||
hashX_pairs = await asyncio.get_event_loop().run_in_executor(self.executor, lookup_hashXs)
|
||||
return await asyncio.get_event_loop().run_in_executor(self.executor, lookup_utxos, hashX_pairs)
|
||||
hashX_pairs = await asyncio.get_event_loop().run_in_executor(None, lookup_hashXs)
|
||||
return await asyncio.get_event_loop().run_in_executor(None, lookup_utxos, hashX_pairs)
|
||||
@@ -6,12 +6,12 @@
|
||||
# and warranty status of this software.
|
||||
|
||||
"""Mempool handling."""
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from prometheus_client import Histogram
|
||||
|
||||
import attr
|
||||
|
||||
@@ -72,22 +72,12 @@ class MemPoolAPI(ABC):
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def on_mempool(self, touched, new_touched, height):
|
||||
async def on_mempool(self, touched, height):
|
||||
"""Called each time the mempool is synchronized. touched is a set of
|
||||
hashXs touched since the previous call. height is the
|
||||
daemon's height at the time the mempool was obtained."""
|
||||
|
||||
|
||||
NAMESPACE = "wallet_server"
|
||||
HISTOGRAM_BUCKETS = (
|
||||
.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf')
|
||||
)
|
||||
mempool_process_time_metric = Histogram(
|
||||
"processed_mempool", "Time to process mempool and notify touched addresses",
|
||||
namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS
|
||||
)
|
||||
|
||||
|
||||
class MemPool:
|
||||
"""Representation of the daemon's mempool.
|
||||
|
||||
@@ -102,7 +92,7 @@ class MemPool:
|
||||
hashXs: hashX -> set of all hashes of txs touching the hashX
|
||||
"""
|
||||
|
||||
def __init__(self, coin, api, refresh_secs=1.0, log_status_secs=120.0):
|
||||
def __init__(self, coin, api, refresh_secs=5.0, log_status_secs=120.0):
|
||||
assert isinstance(api, MemPoolAPI)
|
||||
self.coin = coin
|
||||
self.api = api
|
||||
@@ -115,16 +105,14 @@ class MemPool:
|
||||
# Prevents mempool refreshes during fee histogram calculation
|
||||
self.lock = asyncio.Lock()
|
||||
self.wakeup = asyncio.Event()
|
||||
self.mempool_process_time_metric = mempool_process_time_metric
|
||||
self.notified_mempool_txs = set()
|
||||
|
||||
async def _logging(self, synchronized_event):
|
||||
"""Print regular logs of mempool stats."""
|
||||
self.logger.info('beginning processing of daemon mempool. '
|
||||
'This can take some time...')
|
||||
start = time.perf_counter()
|
||||
start = time.time()
|
||||
await synchronized_event.wait()
|
||||
elapsed = time.perf_counter() - start
|
||||
elapsed = time.time() - start
|
||||
self.logger.info(f'synced in {elapsed:.2f}s')
|
||||
while True:
|
||||
self.logger.info(f'{len(self.txs):,d} txs '
|
||||
@@ -136,7 +124,8 @@ class MemPool:
|
||||
while True:
|
||||
await synchronized_event.wait()
|
||||
async with self.lock:
|
||||
self._update_histogram(100_000)
|
||||
# Threaded as can be expensive
|
||||
await asyncio.get_event_loop().run_in_executor(None, self._update_histogram, 100_000)
|
||||
await asyncio.sleep(self.coin.MEMPOOL_HISTOGRAM_REFRESH_SECS)
|
||||
|
||||
def _update_histogram(self, bin_size):
|
||||
@@ -210,36 +199,19 @@ class MemPool:
|
||||
|
||||
return deferred, {prevout: utxo_map[prevout] for prevout in unspent}
|
||||
|
||||
async def _mempool_loop(self, synchronized_event):
|
||||
try:
|
||||
return await self._refresh_hashes(synchronized_event)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.logger.exception("MEMPOOL DIED")
|
||||
raise e
|
||||
|
||||
async def _refresh_hashes(self, synchronized_event):
|
||||
"""Refresh our view of the daemon's mempool."""
|
||||
while True:
|
||||
start = time.perf_counter()
|
||||
height = self.api.cached_height()
|
||||
hex_hashes = await self.api.mempool_hashes()
|
||||
if height != await self.api.height():
|
||||
continue
|
||||
hashes = {hex_str_to_hash(hh) for hh in hex_hashes}
|
||||
async with self.lock:
|
||||
new_hashes = hashes.difference(self.notified_mempool_txs)
|
||||
touched = await self._process_mempool(hashes)
|
||||
self.notified_mempool_txs.update(new_hashes)
|
||||
new_touched = {
|
||||
touched_hashx for touched_hashx, txs in self.hashXs.items() if txs.intersection(new_hashes)
|
||||
}
|
||||
synchronized_event.set()
|
||||
synchronized_event.clear()
|
||||
await self.api.on_mempool(touched, new_touched, height)
|
||||
duration = time.perf_counter() - start
|
||||
self.mempool_process_time_metric.observe(duration)
|
||||
await self.api.on_mempool(touched, height)
|
||||
try:
|
||||
# we wait up to `refresh_secs` but go early if a broadcast happens (which triggers wakeup event)
|
||||
await asyncio.wait_for(self.wakeup.wait(), timeout=self.refresh_secs)
|
||||
@@ -251,8 +223,7 @@ class MemPool:
|
||||
async def _process_mempool(self, all_hashes):
|
||||
# Re-sync with the new set of hashes
|
||||
txs = self.txs
|
||||
|
||||
hashXs = self.hashXs # hashX: [tx_hash, ...]
|
||||
hashXs = self.hashXs
|
||||
touched = set()
|
||||
|
||||
# First handle txs that have disappeared
|
||||
@@ -283,8 +254,8 @@ class MemPool:
|
||||
# FIXME: this is not particularly efficient
|
||||
while tx_map and len(tx_map) != prior_count:
|
||||
prior_count = len(tx_map)
|
||||
tx_map, utxo_map = self._accept_transactions(tx_map, utxo_map, touched)
|
||||
|
||||
tx_map, utxo_map = self._accept_transactions(tx_map, utxo_map,
|
||||
touched)
|
||||
if tx_map:
|
||||
self.logger.info(f'{len(tx_map)} txs dropped')
|
||||
|
||||
@@ -295,25 +266,30 @@ class MemPool:
|
||||
hex_hashes_iter = (hash_to_hex_str(hash) for hash in hashes)
|
||||
raw_txs = await self.api.raw_transactions(hex_hashes_iter)
|
||||
|
||||
to_hashX = self.coin.hashX_from_script
|
||||
deserializer = self.coin.DESERIALIZER
|
||||
def deserialize_txs(): # This function is pure
|
||||
to_hashX = self.coin.hashX_from_script
|
||||
deserializer = self.coin.DESERIALIZER
|
||||
|
||||
tx_map = {}
|
||||
for hash, raw_tx in zip(hashes, raw_txs):
|
||||
# The daemon may have evicted the tx from its
|
||||
# mempool or it may have gotten in a block
|
||||
if not raw_tx:
|
||||
continue
|
||||
tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
|
||||
# Convert the inputs and outputs into (hashX, value) pairs
|
||||
# Drop generation-like inputs from MemPoolTx.prevouts
|
||||
txin_pairs = tuple((txin.prev_hash, txin.prev_idx)
|
||||
for txin in tx.inputs
|
||||
if not txin.is_generation())
|
||||
txout_pairs = tuple((to_hashX(txout.pk_script), txout.value)
|
||||
for txout in tx.outputs)
|
||||
tx_map[hash] = MemPoolTx(txin_pairs, None, txout_pairs,
|
||||
0, tx_size)
|
||||
txs = {}
|
||||
for hash, raw_tx in zip(hashes, raw_txs):
|
||||
# The daemon may have evicted the tx from its
|
||||
# mempool or it may have gotten in a block
|
||||
if not raw_tx:
|
||||
continue
|
||||
tx, tx_size = deserializer(raw_tx).read_tx_and_vsize()
|
||||
# Convert the inputs and outputs into (hashX, value) pairs
|
||||
# Drop generation-like inputs from MemPoolTx.prevouts
|
||||
txin_pairs = tuple((txin.prev_hash, txin.prev_idx)
|
||||
for txin in tx.inputs
|
||||
if not txin.is_generation())
|
||||
txout_pairs = tuple((to_hashX(txout.pk_script), txout.value)
|
||||
for txout in tx.outputs)
|
||||
txs[hash] = MemPoolTx(txin_pairs, None, txout_pairs,
|
||||
0, tx_size)
|
||||
return txs
|
||||
|
||||
# Thread this potentially slow operation so as not to block
|
||||
tx_map = await asyncio.get_event_loop().run_in_executor(None, deserialize_txs)
|
||||
|
||||
# Determine all prevouts not in the mempool, and fetch the
|
||||
# UTXO information from the database. Failed prevout lookups
|
||||
@@ -324,7 +300,7 @@ class MemPool:
|
||||
for prevout in tx.prevouts
|
||||
if prevout[0] not in all_hashes)
|
||||
utxos = await self.api.lookup_utxos(prevouts)
|
||||
utxo_map = dict(zip(prevouts, utxos))
|
||||
utxo_map = {prevout: utxo for prevout, utxo in zip(prevouts, utxos)}
|
||||
|
||||
return self._accept_transactions(tx_map, utxo_map, touched)
|
||||
|
||||
@@ -335,8 +311,8 @@ class MemPool:
|
||||
async def keep_synchronized(self, synchronized_event):
|
||||
"""Keep the mempool synchronized with the daemon."""
|
||||
await asyncio.wait([
|
||||
self._mempool_loop(synchronized_event),
|
||||
# self._refresh_histogram(synchronized_event),
|
||||
self._refresh_hashes(synchronized_event),
|
||||
self._refresh_histogram(synchronized_event),
|
||||
self._logging(synchronized_event)
|
||||
])
|
||||
|
||||
@@ -353,7 +329,7 @@ class MemPool:
|
||||
value += sum(v for h168, v in tx.out_pairs if h168 == hashX)
|
||||
return value
|
||||
|
||||
def compact_fee_histogram(self):
|
||||
async def compact_fee_histogram(self):
|
||||
"""Return a compact fee histogram of the current mempool."""
|
||||
return self.cached_compact_histogram
|
||||
|
||||
@@ -370,7 +346,7 @@ class MemPool:
|
||||
result.update(tx.prevouts)
|
||||
return result
|
||||
|
||||
def transaction_summaries(self, hashX):
|
||||
async def transaction_summaries(self, hashX):
|
||||
"""Return a list of MemPoolTxSummary objects for the hashX."""
|
||||
result = []
|
||||
for tx_hash in self.hashXs.get(hashX, ()):
|
||||
@@ -393,17 +369,3 @@ class MemPool:
|
||||
if hX == hashX:
|
||||
utxos.append(UTXO(-1, pos, tx_hash, 0, value))
|
||||
return utxos
|
||||
|
||||
def get_mempool_height(self, tx_hash):
|
||||
# Height Progression
|
||||
# -2: not broadcast
|
||||
# -1: in mempool but has unconfirmed inputs
|
||||
# 0: in mempool and all inputs confirmed
|
||||
# +num: confirmed in a specific block (height)
|
||||
if tx_hash not in self.txs:
|
||||
return -2
|
||||
tx = self.txs[tx_hash]
|
||||
unspent_inputs = sum(1 if hash in self.txs else 0 for hash, idx in tx.prevouts)
|
||||
if unspent_inputs:
|
||||
return -1
|
||||
return 0
|
||||
@@ -63,7 +63,7 @@ class Merkle:
|
||||
raise TypeError('index must be an integer')
|
||||
# This also asserts hashes is not empty
|
||||
if not 0 <= index < len(hashes):
|
||||
raise ValueError(f"index '{index}/{len(hashes)}' out of range")
|
||||
raise ValueError('index out of range')
|
||||
natural_length = self.branch_length(len(hashes))
|
||||
if length is None:
|
||||
length = natural_length
|
||||
|
||||
Loaded 100 of 167 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user