Compare commits

..
8 Commits
303 changed files with 24934 additions and 24442 deletions

No files matched your search

-4
View File
@@ -1,13 +1,9 @@
.DS_Store
/build
/dist
/.tox
/.idea
/.coverage
/lbry-venv
lbrynet.egg-info
__pycache__
_trial_temp/
/tests/integration/files
+5 -3
View File
@@ -33,7 +33,9 @@ unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
# extension-pkg-whitelist=
extension-pkg-whitelist=
miniupnpc,
unqlite
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
@@ -352,13 +354,13 @@ ignore-mixin-members=yes
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=leveldb,distutils
ignored-modules=twisted.internet.reactor,leveldb,distutils
# Ignoring distutils because: https://github.com/PyCQA/pylint/issues/73
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set). This supports can work
# with qualified names.
# ignored-classes=
ignored-classes=twisted.internet.reactor,RequestMessage
# List of members which are set dynamically and missed by pylint inference
+27 -42
View File
@@ -3,55 +3,47 @@ dist: xenial
language: python
python: "3.7"
env:
global:
# must also be updated in wine_build.sh
- TORBA=v0.5.4a0
jobs:
include:
- stage: code quality
name: "pylint lbrynet"
install:
- pip install astroid pylint
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install astroid==2.0.4 aiohttp==3.4.4
# newer astroid and aiohttp fails in pylint so we pre-install older version
- pip install pylint
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .
script: pylint lbrynet
- stage: test
- &tests
stage: test
name: "Unit Tests"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .[test]
script:
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -vv tests.unit
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.schema tests.unit.wallet tests.unit.components tests.unit.test_conf
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.test_cli
#- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics
after_success:
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- <<: *tests
name: "DHT Tests"
script: HOME=/tmp coverage run --source=lbrynet -m twisted.trial --reactor=asyncio tests.functional
- name: "Integration Tests"
install:
- pip install tox-travis coverage
- pushd .. && git clone --single-branch --branch ${TORBA} https://github.com/lbryio/torba.git && popd
- pushd .. && git clone https://github.com/lbryio/torba.git && popd
script: tox
after_success:
- coverage combine tests/
- bash <(curl -s https://codecov.io/bash)
- name: "Run Examples"
install:
- pip install coverage
- pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- pip install -e .
script:
- HOME=/tmp coverage run -p --source=lbrynet scripts/generate_json_api.py
after_success:
- coverage combine
- bash <(curl -s https://codecov.io/bash)
- stage: build
name: "Windows"
language: generic
@@ -85,8 +77,8 @@ jobs:
env: OS=linux
install:
- pip3 install pyinstaller
- pip3 install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
- python3 scripts/set_build.py
- pip3 install git+https://github.com/lbryio/torba.git
- python scripts/set_build.py
- pip3 install -e .
script:
- pyinstaller -F -n lbrynet lbrynet/extras/cli.py
@@ -121,22 +113,15 @@ jobs:
before_install:
- brew upgrade python || true
- brew upgrade python || true
- if: tag IS present
stage: build
name: "Wallet Server Docker Image - Tagged Release"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:$TRAVIS_TAG
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
stage: build
name: "Wallet Server Docker Image - Master"
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:master
install:
- python3 --version
- pip3 --version
- pip3 install pyinstaller
- git clone https://github.com/lbryio/torba.git --depth 1
- sed -i -e "s/'plyvel',//" torba/setup.py
- cd torba && pip3 install -e . && cd ..
- python3 scripts/set_build.py
- pip3 install -e .
cache:
directories:
+1 -1
View File
@@ -1,3 +1,3 @@
## Contributing to LBRY
https://lbry.tech/contribute
https://lbry.io/faq/contributing
+12 -11
View File
@@ -13,6 +13,15 @@ Running `lbrynet` from source requires Python 3.6 or higher (3.7 is preferred).
After installing python 3, you'll need to install some additional libraries depending on your operating system.
### Virtualenv
Once python 3 is installed run `python3 -m pip install virtualenv` to install virtualenv.
### Windows
Windows users will need to install `Visual C++ Build Tools`, which can be installed by [Microsoft Build Tools](Microsoft Build Tools 2015)
### macOS
macOS users will need to install [xcode command line tools](https://developer.xamarin.com/guides/testcloud/calabash/configuring/osx/install-xcode-command-line-tools/) and [homebrew](http://brew.sh/).
@@ -20,17 +29,15 @@ macOS users will need to install [xcode command line tools](https://developer.xa
Remaining dependencies can then be installed by running:
```
brew install python protobuf
brew install python3 protobuf
```
Assistance installing Python3: https://docs.python-guide.org/starting/install3/osx/
### Linux
On Ubuntu (we recommend 18.04), install the following:
```
sudo apt-get install build-essential python3.7 python3.7-dev git python3-venv libssl-dev python-protobuf
sudo apt-get install build-essential python3.7 python3.7-dev git python-virtualenv libssl-dev python-protobuf
```
On Raspbian, you will also need to install `python-pyparsing`.
@@ -45,14 +52,8 @@ To install:
git clone https://github.com/lbryio/lbry.git
cd lbry
Creating venv:
python -m venv lbry-venv
Activating lbry-venv on Linux/Mac:
virtualenv lbry-venv --python=python3.7
source lbry-venv/bin/activate
Activating lbry-venv on Windows:
lbry-venv\Scripts\activate
python --version # Python 2 is not supported. Make sure you're on Python 3.7
+1 -1
View File
@@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2015-2019 LBRY Inc
Copyright (c) 2015-2018 LBRY Inc
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
+7 -7
View File
@@ -1,14 +1,14 @@
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [![Build Status](https://travis-ci.org/lbryio/lbry.svg?branch=master)](https://travis-ci.org/lbryio/lbry) [![Test Coverage](https://codecov.io/gh/lbryio/lbry/branch/master/graph/badge.svg)](https://codecov.io/gh/lbryio/lbry)
LBRY is a decentralized peer-to-peer protocol for publishing and accessing digital content. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
* Built on Python 3.7+ and `asyncio`.
* Kademlia DHT (Distributed Hash Table) implementation for finding peers to download from and announcing to peers what we have to host ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for transferring encrypted blobs of content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
* Wallet implementation for the LBRY blockchain ([lbrynet.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/wallet)).
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
## Installation
@@ -33,7 +33,7 @@ Installing from source is also relatively painless. Full instructions are in [IN
## Contributing
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.tech/contribute) link.
Contributions to this project are welcome, encouraged, and compensated. For more details, please check [this](https://lbry.io/faq/contributing) link.
## License
@@ -41,11 +41,11 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
We take security seriously. Please contact security@lbry.io regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
## Contact
The primary contact for this project is [@eukreign](mailto:lex@lbry.com).
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
## Additional information and links
+1533 -2619
View File
File diff suppressed because it is too large. Load diff
+4 -2
View File
@@ -1409,7 +1409,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number (or -1 if unknown),
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1954,6 +1954,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1978,6 +1979,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -2584,4 +2586,4 @@ Returns:
</body>
</html>
</html>
+4 -2
View File
@@ -1339,7 +1339,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number (or -1 if unknown),
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1786,6 +1786,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -1810,6 +1811,7 @@ Returns:
'amount': (float) claim amount,
'effective_amount': (float) claim amount including supports,
'claim_id': (str) claim id,
'claim_sequence': (int) claim sequence number,
'decoded_claim': (bool) whether or not the claim value was decoded,
'height': (int) claim height,
'depth': (int) claim depth,
@@ -2310,4 +2312,4 @@ Returns:
</body>
</html>
</html>
+5 -7
View File
@@ -4,22 +4,20 @@
share_usage_data: True
lbryum_servers:
- lbryumx1.lbry.com:50001
- lbryumx2.lbry.com:50001
- lbryumx4.lbry.com:50001
- lbryumx1.lbry.io:50001
- lbryumx2.lbry.io:50001
blockchain_name: lbrycrd_main
data_dir: /home/lbry/.lbrynet
download_directory: /home/lbry/downloads
save_blobs: true
save_files: false
delete_blobs_on_remove: True
dht_node_port: 4444
peer_port: 3333
use_upnp: true
use_upnp: True
#components_to_skip:
# - peer_protocol_server
# - hash_announcer
# - blob_server
# - dht
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.37.3"
__version__ = "0.30.5"
version = tuple(__version__.split('.'))
+166
View File
@@ -0,0 +1,166 @@
import binascii
import logging
from io import BytesIO
from twisted.internet import defer
from twisted.web.client import FileBodyProducer
from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
from cryptography.hazmat.backends import default_backend
from lbrynet.p2p.BlobInfo import BlobInfo
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
log = logging.getLogger(__name__)
backend = default_backend()
class CryptBlobInfo(BlobInfo):
def __init__(self, blob_hash, blob_num, length, iv):
super().__init__(blob_hash, blob_num, length)
self.iv = iv
def get_dict(self):
info = {
"blob_num": self.blob_num,
"length": self.length,
"iv": self.iv.decode()
}
if self.blob_hash:
info['blob_hash'] = self.blob_hash
return info
class StreamBlobDecryptor:
def __init__(self, blob, key, iv, length):
"""
This class decrypts blob
blob - object which implements read() function.
key = encryption_key
iv = initialization vector
blob_num = blob number (has no effect on encryption)
length = length in bytes of blob
"""
self.blob = blob
self.key = key
self.iv = iv
self.length = length
self.buff = b''
self.len_read = 0
cipher = Cipher(AES(self.key), modes.CBC(self.iv), backend=backend)
self.unpadder = PKCS7(AES.block_size).unpadder()
self.cipher = cipher.decryptor()
def decrypt(self, write_func):
"""
Decrypt blob and write its content using write_func
write_func - function that takes decrypted string as
argument and writes it somewhere
Returns:
deferred that returns after decrypting blob and writing content
"""
def remove_padding(data):
return self.unpadder.update(data) + self.unpadder.finalize()
def write_bytes():
if self.len_read < self.length:
num_bytes_to_decrypt = greatest_multiple(len(self.buff), (AES.block_size // 8))
data_to_decrypt, self.buff = split(self.buff, num_bytes_to_decrypt)
write_func(self.cipher.update(data_to_decrypt))
def finish_decrypt():
bytes_left = len(self.buff) % (AES.block_size // 8)
if bytes_left != 0:
log.warning(binascii.hexlify(self.buff[-1 * (AES.block_size // 8):]).decode())
raise Exception("blob %s has incorrect padding: %i bytes left" %
(self.blob.blob_hash, bytes_left))
data_to_decrypt, self.buff = self.buff, b''
last_chunk = self.cipher.update(data_to_decrypt) + self.cipher.finalize()
write_func(remove_padding(last_chunk))
read_handle = self.blob.open_for_reading()
@defer.inlineCallbacks
def decrypt_bytes():
producer = FileBodyProducer(read_handle)
buff = BytesIO()
yield producer.startProducing(buff)
self.buff = buff.getvalue()
self.len_read += len(self.buff)
write_bytes()
finish_decrypt()
d = decrypt_bytes()
return d
class CryptStreamBlobMaker:
def __init__(self, key, iv, blob_num, blob):
"""
This class encrypts data and writes it to a new blob
key = encryption_key
iv = initialization vector
blob_num = blob number (has no effect on encryption)
blob = object which implements write(), close() function , close() function must
be a deferred. (Will generally be of HashBlobCreator type)
"""
self.key = key
self.iv = iv
self.blob_num = blob_num
self.blob = blob
cipher = Cipher(AES(self.key), modes.CBC(self.iv), backend=backend)
self.padder = PKCS7(AES.block_size).padder()
self.cipher = cipher.encryptor()
self.length = 0
def write(self, data):
"""
encrypt and write string data
Returns:
tuple (done, num_bytes_to_write) where done is True if
max bytes are written. num_bytes_to_write is the number
of bytes that will be written from data in this call
"""
max_bytes_to_write = MAX_BLOB_SIZE - self.length - 1
done = False
if max_bytes_to_write <= len(data):
num_bytes_to_write = max_bytes_to_write
done = True
else:
num_bytes_to_write = len(data)
data_to_write = data[:num_bytes_to_write]
self.length += len(data_to_write)
padded_data = self.padder.update(data_to_write)
encrypted_data = self.cipher.update(padded_data)
self.blob.write(encrypted_data)
return done, num_bytes_to_write
@defer.inlineCallbacks
def close(self):
log.debug("closing blob %s with plaintext len %s", str(self.blob_num), str(self.length))
if self.length != 0:
self.length += (AES.block_size // 8) - (self.length % (AES.block_size // 8))
padded_data = self.padder.finalize()
encrypted_data = self.cipher.update(padded_data) + self.cipher.finalize()
self.blob.write(encrypted_data)
blob_hash = yield self.blob.close()
log.debug("called the finished_callback from CryptStreamBlobMaker.close")
blob = CryptBlobInfo(blob_hash, self.blob_num, self.length, binascii.hexlify(self.iv))
defer.returnValue(blob)
def greatest_multiple(a, b):
"""return the largest value `c`, that is a multiple of `b` and is <= `a`"""
return (a // b) * b
def split(buff, cutoff):
return buff[:cutoff], buff[cutoff:]
+149
View File
@@ -0,0 +1,149 @@
"""
Utility for creating Crypt Streams, which are encrypted blobs and associated metadata.
"""
import os
import logging
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from twisted.internet import defer
from lbrynet.blob.CryptBlob import CryptStreamBlobMaker
log = logging.getLogger(__name__)
class CryptStreamCreator:
"""
Create a new stream with blobs encrypted by a symmetric cipher.
Each blob is encrypted with the same key, but each blob has its
own initialization vector which is associated with the blob when
the blob is associated with the stream.
"""
#implements(interfaces.IConsumer)
def __init__(self, blob_manager, name=None, key=None, iv_generator=None):
"""@param blob_manager: Object that stores and provides access to blobs.
@type blob_manager: BlobManager
@param name: the name of the stream, which will be presented to the user
@type name: string
@param key: the raw AES key which will be used to encrypt the
blobs. If None, a random key will be generated.
@type key: string
@param iv_generator: a generator which yields initialization
vectors for the blobs. Will be called once for each blob.
@type iv_generator: a generator function which yields strings
@return: None
"""
self.blob_manager = blob_manager
self.name = name
self.key = key
if iv_generator is None:
self.iv_generator = self.random_iv_generator()
else:
self.iv_generator = iv_generator
self.stopped = True
self.producer = None
self.streaming = None
self.blob_count = -1
self.current_blob = None
self.finished_deferreds = []
def registerProducer(self, producer, streaming):
from twisted.internet import reactor
self.producer = producer
self.streaming = streaming
self.stopped = False
if streaming is False:
reactor.callLater(0, self.producer.resumeProducing)
def unregisterProducer(self):
self.stopped = True
self.producer = None
def _close_current_blob(self):
# close the blob that was being written to
# and save it to blob manager
should_announce = self.blob_count == 0
d = self.current_blob.close()
d.addCallback(self._blob_finished)
d.addCallback(lambda blob_info: self.blob_manager.creator_finished(blob_info,
should_announce))
self.finished_deferreds.append(d)
self.current_blob = None
def stop(self):
"""Stop creating the stream. Create the terminating zero-length blob."""
log.debug("stop has been called for StreamCreator")
self.stopped = True
if self.current_blob is not None:
self._close_current_blob()
d = self._finalize()
d.addCallback(lambda _: self._finished())
return d
# TODO: move the stream creation process to its own thread and
# remove the reactor from this process.
def write(self, data):
from twisted.internet import reactor
self._write(data)
if self.stopped is False and self.streaming is False:
reactor.callLater(0, self.producer.resumeProducing)
@staticmethod
def random_iv_generator():
while 1:
yield os.urandom(AES.block_size // 8)
def setup(self):
"""Create the symmetric key if it wasn't provided"""
if self.key is None:
self.key = os.urandom(AES.block_size // 8)
return defer.succeed(True)
@defer.inlineCallbacks
def _finalize(self):
"""
Finalize a stream by adding an empty
blob at the end, this is to indicate that
the stream has ended. This empty blob is not
saved to the blob manager
"""
yield defer.DeferredList(self.finished_deferreds)
self.blob_count += 1
iv = next(self.iv_generator)
final_blob = self._get_blob_maker(iv, self.blob_manager.get_blob_creator())
stream_terminator = yield final_blob.close()
terminator_info = yield self._blob_finished(stream_terminator)
defer.returnValue(terminator_info)
def _write(self, data):
while len(data) > 0:
if self.current_blob is None:
self.next_blob_creator = self.blob_manager.get_blob_creator()
self.blob_count += 1
iv = next(self.iv_generator)
self.current_blob = self._get_blob_maker(iv, self.next_blob_creator)
done, num_bytes_written = self.current_blob.write(data)
data = data[num_bytes_written:]
if done is True:
self._close_current_blob()
def _get_blob_maker(self, iv, blob_creator):
return CryptStreamBlobMaker(self.key, iv, self.blob_count, blob_creator)
def _finished(self):
raise NotImplementedError()
def _blob_finished(self, blob_info):
raise NotImplementedError()
+132
View File
@@ -0,0 +1,132 @@
"""
Utilities for turning plain files into LBRY Files.
"""
import os
import logging
from binascii import hexlify
from twisted.internet import defer
from twisted.protocols.basic import FileSender
from lbrynet.extras.compat import f2d
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
from lbrynet.p2p.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
from lbrynet.blob.CryptStreamCreator import CryptStreamCreator
log = logging.getLogger(__name__)
class EncryptedFileStreamCreator(CryptStreamCreator):
"""
A CryptStreamCreator which adds itself and its additional metadata to an EncryptedFileManager
"""
def __init__(self, blob_manager, lbry_file_manager, stream_name=None,
key=None, iv_generator=None):
super().__init__(blob_manager, stream_name, key, iv_generator)
self.lbry_file_manager = lbry_file_manager
self.stream_hash = None
self.blob_infos = []
self.sd_info = None
def _blob_finished(self, blob_info):
log.debug("length: %s", blob_info.length)
self.blob_infos.append(blob_info.get_dict())
return blob_info
def _finished(self):
# calculate the stream hash
self.stream_hash = get_stream_hash(
hexlify(self.name.encode()).decode(), hexlify(self.key).decode(), hexlify(self.name.encode()).decode(),
self.blob_infos
)
# generate the sd info
self.sd_info = format_sd_info(
EncryptedFileStreamType, hexlify(self.name.encode()).decode(), hexlify(self.key).decode(),
hexlify(self.name.encode()).decode(), self.stream_hash, self.blob_infos
)
# sanity check
validate_descriptor(self.sd_info)
return defer.succeed(self.stream_hash)
# TODO: this should be run its own thread. Encrypting a large file can
# be very cpu intensive and there is no need to run that on the
# main reactor thread. The FileSender mechanism that is used is
# great when sending over the network, but this is all local so
# we can simply read the file from the disk without needing to
# involve reactor.
@defer.inlineCallbacks
def create_lbry_file(blob_manager, storage, payment_rate_manager, lbry_file_manager, file_name, file_handle,
key=None, iv_generator=None):
"""Turn a plain file into an LBRY File.
An LBRY File is a collection of encrypted blobs of data and the metadata that binds them
together which, when decrypted and put back together according to the metadata, results
in the original file.
The stream parameters that aren't specified are generated, the file is read and broken
into chunks and encrypted, and then a stream descriptor file with the stream parameters
and other metadata is written to disk.
@param session: An Session object.
@type session: Session
@param lbry_file_manager: The EncryptedFileManager object this LBRY File will be added to.
@type lbry_file_manager: EncryptedFileManager
@param file_name: The path to the plain file.
@type file_name: string
@param file_handle: The file-like object to read
@type file_handle: any file-like object which can be read by twisted.protocols.basic.FileSender
@param key: the raw AES key which will be used to encrypt the blobs. If None, a random key will
be generated.
@type key: string
@param iv_generator: a generator which yields initialization
vectors for the blobs. Will be called once for each blob.
@type iv_generator: a generator function which yields strings
@return: a Deferred which fires with the stream_hash of the LBRY File
@rtype: Deferred which fires with hex-encoded string
"""
base_file_name = os.path.basename(file_name)
file_directory = os.path.dirname(file_handle.name)
lbry_file_creator = EncryptedFileStreamCreator(
blob_manager, lbry_file_manager, base_file_name, key, iv_generator
)
yield lbry_file_creator.setup()
# TODO: Using FileSender isn't necessary, we can just read
# straight from the disk. The stream creation process
# should be in its own thread anyway so we don't need to
# worry about interacting with the twisted reactor
file_sender = FileSender()
yield file_sender.beginFileTransfer(file_handle, lbry_file_creator)
log.debug("the file sender has triggered its deferred. stopping the stream writer")
yield lbry_file_creator.stop()
log.debug("making the sd blob")
sd_info = lbry_file_creator.sd_info
descriptor_writer = BlobStreamDescriptorWriter(blob_manager)
sd_hash = yield descriptor_writer.create_descriptor(sd_info)
log.debug("saving the stream")
yield f2d(storage.store_stream(
sd_info['stream_hash'], sd_hash, sd_info['stream_name'], sd_info['key'],
sd_info['suggested_file_name'], sd_info['blobs']
))
log.debug("adding to the file manager")
lbry_file = yield f2d(lbry_file_manager.add_published_file(
sd_info['stream_hash'], sd_hash, hexlify(file_directory.encode()), payment_rate_manager,
payment_rate_manager.min_blob_data_payment_rate
))
defer.returnValue(lbry_file)
+189
View File
@@ -0,0 +1,189 @@
"""
Download LBRY Files from LBRYnet and save them to disk.
"""
import logging
from binascii import hexlify, unhexlify
from twisted.internet import defer
from lbrynet.conf import Config
from lbrynet.extras.compat import f2d
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
from lbrynet.utils import short_hash
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
from lbrynet.p2p.StreamDescriptor import save_sd_info
log = logging.getLogger(__name__)
def log_status(sd_hash, status):
if status == ManagedEncryptedFileDownloader.STATUS_RUNNING:
status_string = "running"
elif status == ManagedEncryptedFileDownloader.STATUS_STOPPED:
status_string = "stopped"
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
status_string = "finished"
else:
status_string = "unknown"
log.debug("stream %s is %s", short_hash(sd_hash), status_string)
class ManagedEncryptedFileDownloader(EncryptedFileSaver):
STATUS_RUNNING = "running"
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
def __init__(self, conf: Config, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
lbry_file_manager, payment_rate_manager, wallet, download_directory, file_name, stream_name,
sd_hash, key, suggested_file_name, download_mirrors=None):
super().__init__(
conf, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager,
wallet, download_directory, key, stream_name, file_name
)
self.sd_hash = sd_hash
self.rowid = rowid
self.suggested_file_name = unhexlify(suggested_file_name).decode()
self.lbry_file_manager = lbry_file_manager
self._saving_status = False
self.claim_id = None
self.outpoint = None
self.claim_name = None
self.txid = None
self.nout = None
self.channel_claim_id = None
self.channel_name = None
self.metadata = None
self.mirror = None
if download_mirrors or conf.download_mirrors:
self.mirror = HTTPBlobDownloader(
self.blob_manager, servers=download_mirrors or conf.download_mirrors
)
def set_claim_info(self, claim_info):
self.claim_id = claim_info['claim_id']
self.txid = claim_info['txid']
self.nout = claim_info['nout']
self.channel_claim_id = claim_info['channel_claim_id']
self.outpoint = "%s:%i" % (self.txid, self.nout)
self.claim_name = claim_info['name']
self.channel_name = claim_info['channel_name']
self.metadata = claim_info['value']['stream']['metadata']
async def get_claim_info(self, include_supports=True):
claim_info = await self.storage.get_content_claim(self.stream_hash, include_supports)
if claim_info:
self.set_claim_info(claim_info)
return claim_info
@property
def saving_status(self):
return self._saving_status
def restore(self, status):
if status == ManagedEncryptedFileDownloader.STATUS_RUNNING:
# start returns self.finished_deferred
# which fires when we've finished downloading the file
# and we don't want to wait for the entire download
self.start()
elif status == ManagedEncryptedFileDownloader.STATUS_STOPPED:
pass
elif status == ManagedEncryptedFileDownloader.STATUS_FINISHED:
self.completed = True
else:
raise Exception(f"Unknown status for stream {self.stream_hash}: {status}")
@defer.inlineCallbacks
def stop(self, err=None, change_status=True):
log.debug('Stopping download for stream %s', short_hash(self.stream_hash))
if self.mirror:
self.mirror.stop()
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
yield super().stop(err)
if change_status is True:
status = yield self._save_status()
defer.returnValue(status)
async def status(self):
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
completed_blobs = self.blob_manager.completed_blobs(blob_hashes)
num_blobs_completed = len(completed_blobs)
num_blobs_known = len(blob_hashes)
if self.completed:
status = "completed"
elif self.stopped:
status = "stopped"
else:
status = "running"
return EncryptedFileStatusReport(
self.file_name, num_blobs_completed, num_blobs_known, status
)
@defer.inlineCallbacks
def _start(self):
yield EncryptedFileSaver._start(self)
status = yield self._save_status()
log_status(self.sd_hash, status)
if self.mirror:
self.mirror.download_stream(self.stream_hash, self.sd_hash)
defer.returnValue(status)
def _get_finished_deferred_callback_value(self):
if self.completed is True:
return "Download successful"
else:
return "Download stopped"
@defer.inlineCallbacks
def _save_status(self):
self._saving_status = True
if self.completed is True:
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
elif self.stopped is True:
status = ManagedEncryptedFileDownloader.STATUS_STOPPED
else:
status = ManagedEncryptedFileDownloader.STATUS_RUNNING
status = yield self.lbry_file_manager.change_lbry_file_status(self, status)
self._saving_status = False
return status
def save_status(self):
return self._save_status()
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading,
self.blob_manager, download_manager)
class ManagedEncryptedFileDownloaderFactory:
#implements(IStreamDownloaderFactory)
def __init__(self, lbry_file_manager, blob_manager):
self.lbry_file_manager = lbry_file_manager
self.blob_manager = blob_manager
def can_download(self, sd_validator):
# TODO: add a sd_validator for non live streams, use it
return True
@defer.inlineCallbacks
def make_downloader(self, metadata, data_rate, payment_rate_manager, download_directory, file_name=None,
download_mirrors=None):
stream_hash = yield save_sd_info(self.blob_manager,
metadata.source_blob_hash,
metadata.validator.raw_info)
if file_name:
file_name = hexlify(file_name.encode())
hex_download_directory = hexlify(download_directory.encode())
lbry_file = yield f2d(self.lbry_file_manager.add_downloaded_file(
stream_hash, metadata.source_blob_hash, hex_download_directory, payment_rate_manager,
data_rate, file_name=file_name, download_mirrors=download_mirrors
))
defer.returnValue(lbry_file)
@staticmethod
def get_description():
return "Save the file to disk"
+254
View File
@@ -0,0 +1,254 @@
"""
Keep track of which LBRY Files are downloading and store their LBRY File specific metadata
"""
import os
import logging
import random
from binascii import hexlify, unhexlify
from twisted.internet import defer, task, reactor
from twisted.python.failure import Failure
from lbrynet.conf import Config
from lbrynet.extras.compat import f2d
from lbrynet.extras.reflector.reupload import reflect_file
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, get_sd_info
from lbrynet.blob.client.CryptStreamDownloader import AlreadyStoppedError
from lbrynet.blob.client.CryptStreamDownloader import CurrentlyStoppingError
from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
log = logging.getLogger(__name__)
class EncryptedFileManager:
"""
Keeps track of currently opened LBRY Files, their options, and
their LBRY File specific metadata.
"""
# when reflecting files, reflect up to this many files at a time
CONCURRENT_REFLECTS = 5
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, wallet,
payment_rate_manager, storage, sd_identifier):
self.conf = conf
self.auto_re_reflect = conf.reflect_uploads and conf.auto_re_reflect_interval > 0
self.auto_re_reflect_interval = conf.auto_re_reflect_interval
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.blob_manager = blob_manager
self.wallet = wallet
self.payment_rate_manager = payment_rate_manager
self.storage = storage
# TODO: why is sd_identifier part of the file manager?
self.sd_identifier = sd_identifier
self.lbry_files = []
self.lbry_file_reflector = task.LoopingCall(self.reflect_lbry_files)
def setup(self):
self._add_to_sd_identifier()
return self._start_lbry_files()
def get_lbry_file_status(self, lbry_file):
return self.storage.get_lbry_file_status(lbry_file.rowid)
def set_lbry_file_data_payment_rate(self, lbry_file, new_rate):
return self.storage(lbry_file.rowid, new_rate)
def change_lbry_file_status(self, lbry_file, status):
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
return f2d(self.storage.change_file_status(lbry_file.rowid, status))
def get_lbry_file_status_reports(self):
ds = []
for lbry_file in self.lbry_files:
ds.append(lbry_file.status())
dl = defer.DeferredList(ds)
def filter_failures(status_reports):
return [status_report for success, status_report in status_reports if success is True]
dl.addCallback(filter_failures)
return dl
def _add_to_sd_identifier(self):
downloader_factory = ManagedEncryptedFileDownloaderFactory(self, self.blob_manager)
self.sd_identifier.add_stream_downloader_factory(
EncryptedFileStreamType, downloader_factory)
def _get_lbry_file(self, rowid, stream_hash, payment_rate_manager, sd_hash, key,
stream_name, file_name, download_directory, suggested_file_name, download_mirrors=None):
return ManagedEncryptedFileDownloader(
self.conf,
rowid,
stream_hash,
self.peer_finder,
self.rate_limiter,
self.blob_manager,
self.storage,
self,
payment_rate_manager,
self.wallet,
download_directory,
file_name,
stream_name=stream_name,
sd_hash=sd_hash,
key=key,
suggested_file_name=suggested_file_name,
download_mirrors=download_mirrors
)
def _start_lbry_file(self, file_info, payment_rate_manager, claim_info, download_mirrors=None):
lbry_file = self._get_lbry_file(
file_info['row_id'], file_info['stream_hash'], payment_rate_manager, file_info['sd_hash'],
file_info['key'], file_info['stream_name'], file_info['file_name'], file_info['download_directory'],
file_info['suggested_file_name'], download_mirrors
)
if claim_info:
lbry_file.set_claim_info(claim_info)
try:
# restore will raise an Exception if status is unknown
lbry_file.restore(file_info['status'])
self.storage.content_claim_callbacks[lbry_file.stream_hash] = lbry_file.get_claim_info
self.lbry_files.append(lbry_file)
if len(self.lbry_files) % 500 == 0:
log.info("Started %i files", len(self.lbry_files))
except Exception:
log.warning("Failed to start %i", file_info.get('rowid'))
async def _start_lbry_files(self):
files = await self.storage.get_all_lbry_files()
claim_infos = await self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
prm = self.payment_rate_manager
log.info("Starting %i files", len(files))
for file_info in files:
claim_info = claim_infos.get(file_info['stream_hash'])
self._start_lbry_file(file_info, prm, claim_info)
log.info("Started %i lbry files", len(self.lbry_files))
if self.auto_re_reflect is True:
safe_start_looping_call(self.lbry_file_reflector, self.auto_re_reflect_interval / 10)
@defer.inlineCallbacks
def _stop_lbry_file(self, lbry_file):
def wait_for_finished(lbry_file, count=2):
if count or lbry_file.saving_status is not False:
return task.deferLater(reactor, 1, self._stop_lbry_file, lbry_file,
count=count - 1)
try:
yield lbry_file.stop(change_status=False)
self.lbry_files.remove(lbry_file)
except CurrentlyStoppingError:
yield wait_for_finished(lbry_file)
except AlreadyStoppedError:
pass
finally:
defer.returnValue(None)
@defer.inlineCallbacks
def _stop_lbry_files(self):
log.info("Stopping %i lbry files", len(self.lbry_files))
yield defer.DeferredList([self._stop_lbry_file(lbry_file) for lbry_file in list(self.lbry_files)])
async def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
key = stream_metadata['key']
stream_name = stream_metadata['stream_name']
file_name = stream_metadata['suggested_file_name']
rowid = await self.storage.save_published_file(
stream_hash, file_name, download_directory, blob_data_rate, status
)
lbry_file = self._get_lbry_file(
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
stream_metadata['suggested_file_name'], download_mirrors=None
)
lbry_file.restore(status)
await lbry_file.get_claim_info()
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
self.lbry_files.append(lbry_file)
return lbry_file
async def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
blob_data_rate=None, status=None, file_name=None, download_mirrors=None):
status = status or ManagedEncryptedFileDownloader.STATUS_STOPPED
payment_rate_manager = payment_rate_manager or self.payment_rate_manager
blob_data_rate = blob_data_rate or payment_rate_manager.min_blob_data_payment_rate
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
key = stream_metadata['key']
stream_name = stream_metadata['stream_name']
file_name = file_name or stream_metadata['suggested_file_name']
# when we save the file we'll atomic touch the nearest file to the suggested file name
# that doesn't yet exist in the download directory
rowid = await self.storage.save_downloaded_file(
stream_hash, hexlify(os.path.basename(unhexlify(file_name))), download_directory, blob_data_rate
)
file_name = (await self.storage.get_filename_for_rowid(rowid)).decode()
lbry_file = self._get_lbry_file(
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
stream_metadata['suggested_file_name'], download_mirrors
)
lbry_file.restore(status)
await lbry_file.get_claim_info(include_supports=False)
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
self.lbry_files.append(lbry_file)
return lbry_file
@defer.inlineCallbacks
def delete_lbry_file(self, lbry_file, delete_file=False):
if lbry_file not in self.lbry_files:
raise ValueError("Could not find that LBRY file")
def wait_for_finished(count=2):
if count <= 0 or lbry_file.saving_status is False:
return True
else:
return task.deferLater(reactor, 1, wait_for_finished, count=count - 1)
full_path = os.path.join(lbry_file.download_directory, lbry_file.file_name)
try:
yield lbry_file.stop()
except (AlreadyStoppedError, CurrentlyStoppingError):
yield wait_for_finished()
self.lbry_files.remove(lbry_file)
if lbry_file.stream_hash in self.storage.content_claim_callbacks:
del self.storage.content_claim_callbacks[lbry_file.stream_hash]
yield lbry_file.delete_data()
yield f2d(self.storage.delete_stream(lbry_file.stream_hash))
if delete_file and os.path.isfile(full_path):
os.remove(full_path)
defer.returnValue(True)
def toggle_lbry_file_running(self, lbry_file):
"""Toggle whether a stream reader is currently running"""
for l in self.lbry_files:
if l == lbry_file:
return l.toggle_running()
return defer.fail(Failure(ValueError("Could not find that LBRY file")))
@defer.inlineCallbacks
def reflect_lbry_files(self):
sem = defer.DeferredSemaphore(self.CONCURRENT_REFLECTS)
ds = []
sd_hashes_to_reflect = yield f2d(self.storage.get_streams_to_re_reflect())
for lbry_file in self.lbry_files:
if lbry_file.sd_hash in sd_hashes_to_reflect:
ds.append(sem.run(reflect_file, lbry_file, random.choice(self.conf.reflector_servers)))
yield defer.DeferredList(ds)
@defer.inlineCallbacks
def stop(self):
safe_stop_looping_call(self.lbry_file_reflector)
yield self._stop_lbry_files()
log.info("Stopped encrypted file manager")
defer.returnValue(True)
@@ -0,0 +1,6 @@
class EncryptedFileStatusReport:
def __init__(self, name, num_completed, num_known, running_status):
self.name = name
self.num_completed = num_completed
self.num_known = num_known
self.running_status = running_status
-6
View File
@@ -1,6 +0,0 @@
from lbrynet.cryptoutils import get_lbry_hash_obj
MAX_BLOB_SIZE = 2 * 2 ** 20
# digest_size is in bytes, and blob hashes are hex encoded
blobhash_length = get_lbry_hash_obj().digest_size * 2
+174 -292
View File
@@ -1,29 +1,26 @@
import os
import re
import asyncio
import binascii
import logging
import typing
import contextlib
from io import BytesIO
from cryptography.hazmat.primitives.ciphers import Cipher, modes
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from cryptography.hazmat.primitives.padding import PKCS7
from lbrynet.cryptoutils import backend, get_lbry_hash_obj
from lbrynet.error import DownloadCancelledError, InvalidBlobHashError, InvalidDataError
from lbrynet.blob import MAX_BLOB_SIZE, blobhash_length
from lbrynet.blob.blob_info import BlobInfo
from twisted.internet import defer
from twisted.web.client import FileBodyProducer
from twisted.python.failure import Failure
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
from lbrynet.blob.writer import HashBlobWriter
from lbrynet.blob.reader import HashBlobReader
log = logging.getLogger(__name__)
MAX_BLOB_SIZE = 2 * 2 ** 20
_hexmatch = re.compile("^[a-f,0-9]+$")
# digest_size is in bytes, and blob hashes are hex encoded
blobhash_length = get_lbry_hash_obj().digest_size * 2
def is_valid_blobhash(blobhash: str) -> bool:
def is_valid_hashcharacter(char):
return char in "0123456789abcdef"
def is_valid_blobhash(blobhash):
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
@@ -31,305 +28,190 @@ def is_valid_blobhash(blobhash: str) -> bool:
@return: True/False
"""
return len(blobhash) == blobhash_length and _hexmatch.match(blobhash)
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
def encrypt_blob_bytes(key: bytes, iv: bytes, unencrypted: bytes) -> typing.Tuple[bytes, str]:
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
padder = PKCS7(AES.block_size).padder()
encryptor = cipher.encryptor()
encrypted = encryptor.update(padder.update(unencrypted) + padder.finalize()) + encryptor.finalize()
digest = get_lbry_hash_obj()
digest.update(encrypted)
return encrypted, digest.hexdigest()
def decrypt_blob_bytes(data: bytes, length: int, key: bytes, iv: bytes) -> bytes:
if len(data) != length:
raise ValueError("unexpected length")
cipher = Cipher(AES(key), modes.CBC(iv), backend=backend)
unpadder = PKCS7(AES.block_size).unpadder()
decryptor = cipher.decryptor()
return unpadder.update(decryptor.update(data) + decryptor.finalize()) + unpadder.finalize()
class AbstractBlob:
class BlobFile:
"""
A chunk of data (up to 2MB) available on the network which is specified by a sha384 hash
A chunk of data available on the network which is specified by a hashsum
This class is non-io specific
This class is used to create blobs on the local filesystem
when we already know the blob hash before hand (i.e., when downloading blobs)
Also can be used for reading from blobs on the local filesystem
"""
__slots__ = [
'loop',
'blob_hash',
'length',
'blob_completed_callback',
'blob_directory',
'writers',
'verified',
'writing',
'readers'
]
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
self.loop = loop
self.blob_hash = blob_hash
self.length = length
self.blob_completed_callback = blob_completed_callback
self.blob_directory = blob_directory
self.writers: typing.Dict[typing.Tuple[typing.Optional[str], typing.Optional[int]], HashBlobWriter] = {}
self.verified: asyncio.Event = asyncio.Event(loop=self.loop)
self.writing: asyncio.Event = asyncio.Event(loop=self.loop)
self.readers: typing.List[typing.BinaryIO] = []
def __str__(self):
return self.blob_hash[:16]
def __repr__(self):
return '<{}({})>'.format(self.__class__.__name__, str(self))
def __init__(self, blob_dir, blob_hash, length=None):
if not is_valid_blobhash(blob_hash):
raise InvalidBlobHashError(blob_hash)
self.blob_hash = blob_hash
self.length = length
self.writers = {} # {Peer: writer, finished_deferred}
self._verified = False
self.readers = 0
self.blob_dir = blob_dir
self.file_path = os.path.join(blob_dir, self.blob_hash)
self.blob_write_lock = defer.DeferredLock()
self.saved_verified_blob = False
if os.path.isfile(self.file_path):
self.set_length(os.path.getsize(self.file_path))
# This assumes that the hash of the blob has already been
# checked as part of the blob creation process. It might
# be worth having a function that checks the actual hash;
# its probably too expensive to have that check be part of
# this call.
self._verified = True
def __del__(self):
if self.writers or self.readers:
log.warning("%s not closed before being garbage collected", self.blob_hash)
self.close()
def open_for_writing(self, peer):
"""
open a blob file to be written by peer, supports concurrent
writers, as long as they are from different peers.
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
raise NotImplementedError()
returns tuple of (writer, finished_deferred)
@contextlib.contextmanager
def reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
if not self.is_readable():
raise OSError(f"{str(type(self))} not readable, {len(self.readers)} readers {len(self.writers)} writers")
with self._reader_context() as reader:
try:
self.readers.append(reader)
yield reader
finally:
if reader in self.readers:
self.readers.remove(reader)
writer - a file like object with a write() function, close() when finished
finished_deferred - deferred that is fired when write is finished and returns
a instance of itself as HashBlob
"""
if peer not in self.writers:
log.debug("Opening %s to be written by %s", str(self), str(peer))
finished_deferred = defer.Deferred()
writer = HashBlobWriter(self.get_length, self.writer_finished)
self.writers[peer] = (writer, finished_deferred)
return writer, finished_deferred
log.warning("Tried to download the same file twice simultaneously from the same peer")
return None, None
def _write_blob(self, blob_bytes: bytes):
raise NotImplementedError()
def open_for_reading(self):
"""
open blob for reading
def set_length(self, length) -> None:
if self.length is not None and length == self.length:
return
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
self.length = length
return
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s", self.length, length)
def get_length(self) -> typing.Optional[int]:
return self.length
def get_is_verified(self) -> bool:
return self.verified.is_set()
def is_readable(self) -> bool:
return self.verified.is_set()
def is_writeable(self) -> bool:
return not self.writing.is_set()
def write_blob(self, blob_bytes: bytes):
if not self.is_writeable():
raise OSError("cannot open blob for writing")
try:
self.writing.set()
self._write_blob(blob_bytes)
finally:
self.writing.clear()
def close(self):
while self.writers:
peer, writer = self.writers.popitem()
if writer and writer.finished and not writer.finished.done() and not self.loop.is_closed():
writer.finished.cancel()
while self.readers:
reader = self.readers.pop()
if reader:
reader.close()
returns a file like object that can be read() from, and closed() when
finished
"""
if self._verified is True:
f = open(self.file_path, 'rb')
reader = HashBlobReader(f, self.reader_finished)
self.readers += 1
return reader
return None
def delete(self):
self.close()
self.verified.clear()
self.length = None
async def sendfile(self, writer: asyncio.StreamWriter) -> int:
"""
Read and send the file to the writer and return the number of bytes sent
delete blob file from file system, prevent deletion
if a blob is being read from or written to
returns a deferred that firesback when delete is completed
"""
if not self.is_readable():
raise OSError('blob files cannot be read')
with self.reader_context() as handle:
return await self.loop.sendfile(writer.transport, handle, count=self.get_length())
def decrypt(self, key: bytes, iv: bytes) -> bytes:
"""
Decrypt a BlobFile to plaintext bytes
"""
with self.reader_context() as reader:
return decrypt_blob_bytes(reader.read(), self.length, key, iv)
@classmethod
async def create_from_unencrypted(
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], None]] = None) -> BlobInfo:
"""
Create an encrypted BlobFile from plaintext bytes
"""
blob_bytes, blob_hash = encrypt_blob_bytes(key, iv, unencrypted)
length = len(blob_bytes)
blob = cls(loop, blob_hash, length, blob_completed_callback, blob_dir)
writer = blob.get_blob_writer()
writer.write(blob_bytes)
await blob.verified.wait()
return BlobInfo(blob_num, length, binascii.hexlify(iv).decode(), blob_hash)
def save_verified_blob(self, verified_bytes: bytes):
if self.verified.is_set():
return
if self.is_writeable():
self._write_blob(verified_bytes)
self.verified.set()
if self.blob_completed_callback:
self.blob_completed_callback(self)
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[int] = None) -> HashBlobWriter:
if (peer_address, peer_port) in self.writers and not self.writers[(peer_address, peer_port)].closed():
raise OSError(f"attempted to download blob twice from {peer_address}:{peer_port}")
fut = asyncio.Future(loop=self.loop)
writer = HashBlobWriter(self.blob_hash, self.get_length, fut)
self.writers[(peer_address, peer_port)] = writer
def remove_writer(_):
if (peer_address, peer_port) in self.writers:
del self.writers[(peer_address, peer_port)]
fut.add_done_callback(remove_writer)
def writer_finished_callback(finished: asyncio.Future):
if not self.writers and not self.readers:
self._verified = False
self.saved_verified_blob = False
try:
err = finished.exception()
if err:
raise err
verified_bytes = finished.result()
while self.writers:
_, other = self.writers.popitem()
if other is not writer:
other.close_handle()
self.save_verified_blob(verified_bytes)
except (InvalidBlobHashError, InvalidDataError) as error:
log.warning("writer error downloading %s: %s", self.blob_hash[:8], str(error))
except (DownloadCancelledError, asyncio.CancelledError, asyncio.TimeoutError):
pass
fut.add_done_callback(writer_finished_callback)
return writer
class BlobBuffer(AbstractBlob):
"""
An in-memory only blob
"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
self._verified_bytes: typing.Optional[BytesIO] = None
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
if not self.is_readable():
raise OSError("cannot open blob for reading")
try:
yield self._verified_bytes
finally:
if self._verified_bytes:
self._verified_bytes.close()
self._verified_bytes = None
self.verified.clear()
def _write_blob(self, blob_bytes: bytes):
if self._verified_bytes:
raise OSError("already have bytes for blob")
self._verified_bytes = BytesIO(blob_bytes)
def delete(self):
if self._verified_bytes:
self._verified_bytes.close()
self._verified_bytes = None
return super().delete()
def __del__(self):
super().__del__()
if self._verified_bytes:
self.delete()
class BlobFile(AbstractBlob):
"""
A blob existing on the local file system
"""
def __init__(self, loop: asyncio.BaseEventLoop, blob_hash: str, length: typing.Optional[int] = None,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'], asyncio.Task]] = None,
blob_directory: typing.Optional[str] = None):
super().__init__(loop, blob_hash, length, blob_completed_callback, blob_directory)
if not blob_directory or not os.path.isdir(blob_directory):
raise OSError(f"invalid blob directory '{blob_directory}'")
self.file_path = os.path.join(self.blob_directory, self.blob_hash)
if self.file_exists:
file_size = int(os.stat(self.file_path).st_size)
if length and length != file_size:
log.warning("expected %s to be %s bytes, file has %s", self.blob_hash, length, file_size)
self.delete()
else:
self.length = file_size
self.verified.set()
if os.path.isfile(self.file_path):
os.remove(self.file_path)
except Exception as e:
log.exception("An error occurred deleting %s:", str(self.file_path), exc_info=e)
else:
raise ValueError("File is currently being read or written and cannot be deleted")
@property
def file_exists(self):
return os.path.isfile(self.file_path)
def verified(self):
"""
Protect verified from being modified by other classes.
verified is True if a write to a blob has completed successfully,
or a blob has been read to have the same length as specified
in init
"""
return self._verified
def is_writeable(self) -> bool:
return super().is_writeable() and not os.path.isfile(self.file_path)
def set_length(self, length):
if self.length is not None and length == self.length:
return True
if self.length is None and 0 <= length <= MAX_BLOB_SIZE:
self.length = length
return True
log.warning("Got an invalid length. Previous length: %s, Invalid length: %s",
self.length, length)
return False
def get_blob_writer(self, peer_address: typing.Optional[str] = None,
peer_port: typing.Optional[str] = None) -> HashBlobWriter:
if self.file_exists:
raise OSError(f"File already exists '{self.file_path}'")
return super().get_blob_writer(peer_address, peer_port)
def get_length(self):
return self.length
@contextlib.contextmanager
def _reader_context(self) -> typing.ContextManager[typing.BinaryIO]:
handle = open(self.file_path, 'rb')
try:
yield handle
finally:
handle.close()
def get_is_verified(self):
return self.verified
def _write_blob(self, blob_bytes: bytes):
with open(self.file_path, 'wb') as f:
f.write(blob_bytes)
def is_downloading(self):
if self.writers:
return True
return False
def delete(self):
if os.path.isfile(self.file_path):
os.remove(self.file_path)
return super().delete()
def reader_finished(self, reader):
self.readers -= 1
return defer.succeed(True)
@classmethod
async def create_from_unencrypted(
cls, loop: asyncio.BaseEventLoop, blob_dir: typing.Optional[str], key: bytes, iv: bytes,
unencrypted: bytes, blob_num: int,
blob_completed_callback: typing.Optional[typing.Callable[['AbstractBlob'],
asyncio.Task]] = None) -> BlobInfo:
if not blob_dir or not os.path.isdir(blob_dir):
raise OSError(f"cannot create blob in directory: '{blob_dir}'")
return await super().create_from_unencrypted(
loop, blob_dir, key, iv, unencrypted, blob_num, blob_completed_callback
)
def writer_finished(self, writer, err=None):
def fire_finished_deferred():
self._verified = True
for p, (w, finished_deferred) in list(self.writers.items()):
if w == writer:
del self.writers[p]
finished_deferred.callback(self)
return True
log.warning(
"Somehow, the writer that was accepted as being valid was already removed: %s",
writer)
return False
def errback_finished_deferred(err):
for p, (w, finished_deferred) in list(self.writers.items()):
if w == writer:
del self.writers[p]
finished_deferred.errback(err)
def cancel_other_downloads():
for p, (w, finished_deferred) in self.writers.items():
w.close()
if err is None:
if writer.len_so_far == self.length and writer.blob_hash == self.blob_hash:
if self._verified is False:
d = self.save_verified_blob(writer)
d.addCallbacks(lambda _: fire_finished_deferred(), errback_finished_deferred)
d.addCallback(lambda _: cancel_other_downloads())
else:
d = defer.succeed(None)
fire_finished_deferred()
else:
if writer.len_so_far != self.length:
err_string = "blob length is %i vs expected %i" % (writer.len_so_far, self.length)
else:
err_string = f"blob hash is {writer.blob_hash} vs expected {self.blob_hash}"
errback_finished_deferred(Failure(InvalidDataError(err_string)))
d = defer.succeed(None)
else:
errback_finished_deferred(err)
d = defer.succeed(None)
d.addBoth(lambda _: writer.close_handle())
return d
def save_verified_blob(self, writer):
# we cannot have multiple _save_verified_blob interrupting
# each other, can happen since startProducing is a deferred
return self.blob_write_lock.run(self._save_verified_blob, writer)
@defer.inlineCallbacks
def _save_verified_blob(self, writer):
if self.saved_verified_blob is False:
writer.write_handle.seek(0)
out_path = os.path.join(self.blob_dir, self.blob_hash)
producer = FileBodyProducer(writer.write_handle)
yield producer.startProducing(open(out_path, 'wb'))
self.saved_verified_blob = True
defer.returnValue(True)
else:
raise DownloadCanceledError()
-26
View File
@@ -1,26 +0,0 @@
import typing
class BlobInfo:
__slots__ = [
'blob_hash',
'blob_num',
'length',
'iv',
]
def __init__(self, blob_num: int, length: int, iv: str, blob_hash: typing.Optional[str] = None):
self.blob_hash = blob_hash
self.blob_num = blob_num
self.length = length
self.iv = iv
def as_dict(self) -> typing.Dict:
d = {
'length': self.length,
'blob_num': self.blob_num,
'iv': self.iv,
}
if self.blob_hash: # non-terminator blobs have a blob hash
d['blob_hash'] = self.blob_hash
return d
-135
View File
@@ -1,135 +0,0 @@
import os
import typing
import asyncio
import logging
from lbrynet.utils import LRUCache
from lbrynet.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.connection_manager import ConnectionManager
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.protocol.data_store import DictDataStore
from lbrynet.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
class BlobManager:
def __init__(self, loop: asyncio.BaseEventLoop, blob_dir: str, storage: 'SQLiteStorage', config: 'Config',
node_data_store: typing.Optional['DictDataStore'] = None):
"""
This class stores blobs on the hard disk
blob_dir - directory where blobs are stored
storage - SQLiteStorage object
"""
self.loop = loop
self.blob_dir = blob_dir
self.storage = storage
self._node_data_store = node_data_store
self.completed_blob_hashes: typing.Set[str] = set() if not self._node_data_store\
else self._node_data_store.completed_blobs
self.blobs: typing.Dict[str, AbstractBlob] = {}
self.config = config
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
self.config.blob_lru_cache_size)
self.connection_manager = ConnectionManager(loop)
def _get_blob(self, blob_hash: str, length: typing.Optional[int] = None):
if self.config.save_blobs:
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
else:
if is_valid_blobhash(blob_hash) and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return BlobFile(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
return BlobBuffer(
self.loop, blob_hash, length, self.blob_completed, self.blob_dir
)
def get_blob(self, blob_hash, length: typing.Optional[int] = None):
if blob_hash in self.blobs:
if self.config.save_blobs and isinstance(self.blobs[blob_hash], BlobBuffer):
buffer = self.blobs.pop(blob_hash)
if blob_hash in self.completed_blob_hashes:
self.completed_blob_hashes.remove(blob_hash)
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
if buffer.is_readable():
with buffer.reader_context() as reader:
self.blobs[blob_hash].write_blob(reader.read())
if length and self.blobs[blob_hash].length is None:
self.blobs[blob_hash].set_length(length)
else:
self.blobs[blob_hash] = self._get_blob(blob_hash, length)
return self.blobs[blob_hash]
def is_blob_verified(self, blob_hash: str, length: typing.Optional[int] = None) -> bool:
if not is_valid_blobhash(blob_hash):
raise ValueError(blob_hash)
if blob_hash in self.blobs:
return self.blobs[blob_hash].get_is_verified()
if not os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
return False
return self._get_blob(blob_hash, length).get_is_verified()
async def setup(self) -> bool:
def get_files_in_blob_dir() -> typing.Set[str]:
if not self.blob_dir:
return set()
return {
item.name for item in os.scandir(self.blob_dir) if is_valid_blobhash(item.name)
}
in_blobfiles_dir = await self.loop.run_in_executor(None, get_files_in_blob_dir)
to_add = await self.storage.sync_missing_blobs(in_blobfiles_dir)
if to_add:
self.completed_blob_hashes.update(to_add)
self.connection_manager.start()
return True
def stop(self):
self.connection_manager.stop()
while self.blobs:
_, blob = self.blobs.popitem()
blob.close()
self.completed_blob_hashes.clear()
def get_stream_descriptor(self, sd_hash):
return StreamDescriptor.from_stream_descriptor_blob(self.loop, self.blob_dir, self.get_blob(sd_hash))
def blob_completed(self, blob: AbstractBlob) -> asyncio.Task:
if blob.blob_hash is None:
raise Exception("Blob hash is None")
if not blob.length:
raise Exception("Blob has a length of 0")
if isinstance(blob, BlobFile):
if blob.blob_hash not in self.completed_blob_hashes:
self.completed_blob_hashes.add(blob.blob_hash)
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=True))
else:
return self.loop.create_task(self.storage.add_blobs((blob.blob_hash, blob.length), finished=False))
def check_completed_blobs(self, blob_hashes: typing.List[str]) -> typing.List[str]:
"""Returns of the blobhashes_to_check, which are valid"""
return [blob_hash for blob_hash in blob_hashes if self.is_blob_verified(blob_hash)]
def delete_blob(self, blob_hash: str):
if not is_valid_blobhash(blob_hash):
raise Exception("invalid blob hash to delete")
if blob_hash not in self.blobs:
if self.blob_dir and os.path.isfile(os.path.join(self.blob_dir, blob_hash)):
os.remove(os.path.join(self.blob_dir, blob_hash))
else:
self.blobs.pop(blob_hash).delete()
if blob_hash in self.completed_blob_hashes:
self.completed_blob_hashes.remove(blob_hash)
async def delete_blobs(self, blob_hashes: typing.List[str], delete_from_db: typing.Optional[bool] = True):
for blob_hash in blob_hashes:
self.delete_blob(blob_hash)
if delete_from_db:
await self.storage.delete_blobs_from_db(blob_hashes)
+22
View File
@@ -0,0 +1,22 @@
import binascii
from twisted.internet import defer
from lbrynet.blob.CryptBlob import StreamBlobDecryptor
class CryptBlobHandler:
#implements(IBlobHandler)
def __init__(self, key, write_func):
self.key = key
self.write_func = write_func
######## IBlobHandler #########
def handle_blob(self, blob, blob_info):
try:
blob_decryptor = StreamBlobDecryptor(blob, self.key, binascii.unhexlify(blob_info.iv),
blob_info.length)
except ValueError as err:
return defer.fail(err)
d = blob_decryptor.decrypt(self.write_func)
return d
@@ -0,0 +1,221 @@
import logging
from binascii import unhexlify
from twisted.internet import defer
from twisted.python.failure import Failure
from lbrynet.conf import Config
from lbrynet.p2p.client.BlobRequester import BlobRequester
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
from lbrynet.p2p.client.DownloadManager import DownloadManager
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler
log = logging.getLogger(__name__)
class StartFailedError(Exception):
pass
class AlreadyRunningError(Exception):
pass
class AlreadyStoppedError(Exception):
pass
class CurrentlyStoppingError(Exception):
pass
class CurrentlyStartingError(Exception):
pass
class CryptStreamDownloader:
#implements(IStreamDownloader)
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
key, stream_name):
"""Initialize a CryptStreamDownloader
@param peer_finder: An object which implements the IPeerFinder
interface. Used to look up peers by a hashsum.
@param rate_limiter: An object which implements the IRateLimiter interface
@param blob_manager: A BlobManager object
@param payment_rate_manager: A NegotiatedPaymentRateManager object
@param wallet: An object which implements the IWallet interface
@return:
"""
self.conf = conf
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.blob_manager = blob_manager
self.payment_rate_manager = payment_rate_manager
self.wallet = wallet
self.key = unhexlify(key)
self.stream_name = unhexlify(stream_name).decode()
self.completed = False
self.stopped = True
self.stopping = False
self.starting = False
self.download_manager = None
self.finished_deferred = None
self.points_paid = 0.0
self.blob_requester = None
def __str__(self):
return str(self.stream_name)
def toggle_running(self):
if self.stopped is True:
return self.start()
else:
return self.stop()
def start(self):
if self.starting is True:
raise CurrentlyStartingError()
if self.stopping is True:
raise CurrentlyStoppingError()
if self.stopped is False:
raise AlreadyRunningError()
assert self.download_manager is None
self.starting = True
self.completed = False
self.finished_deferred = defer.Deferred()
d = self._start()
d.addCallback(lambda _: self.finished_deferred)
return d
@defer.inlineCallbacks
def stop(self, err=None):
if self.stopped is True:
raise AlreadyStoppedError()
if self.stopping is True:
raise CurrentlyStoppingError()
assert self.download_manager is not None
self.stopping = True
success = yield self.download_manager.stop_downloading()
self.stopping = False
if success is True:
self.stopped = True
self._remove_download_manager()
yield self._fire_completed_deferred(err)
def _start_failed(self):
def set_stopped():
self.stopped = True
self.stopping = False
self.starting = False
if self.download_manager is not None:
d = self.download_manager.stop_downloading()
d.addCallback(lambda _: self._remove_download_manager())
else:
d = defer.succeed(True)
d.addCallback(lambda _: set_stopped())
d.addCallback(lambda _: Failure(StartFailedError()))
return d
def _start(self):
def check_start_succeeded(success):
if success:
self.starting = False
self.stopped = False
self.completed = False
return True
else:
return self._start_failed()
self.download_manager = self._get_download_manager()
d = self.download_manager.start_downloading()
d.addCallbacks(check_start_succeeded)
return d
def _get_download_manager(self):
assert self.blob_requester is None
download_manager = DownloadManager(self.blob_manager)
# TODO: can we get rid of these circular references. I'm not
# smart enough to handle thinking about the interactions
# between them and have hope that there is a simpler way
# to accomplish what we want
download_manager.blob_info_finder = self._get_metadata_handler(download_manager)
download_manager.progress_manager = self._get_progress_manager(download_manager)
download_manager.blob_handler = self._get_blob_handler(download_manager)
download_manager.wallet_info_exchanger = self.wallet.get_info_exchanger()
# blob_requester needs to be set before the connection manager is setup
self.blob_requester = self._get_blob_requester(download_manager)
download_manager.connection_manager = self._get_connection_manager(download_manager)
return download_manager
def _remove_download_manager(self):
self.download_manager.blob_info_finder = None
self.download_manager.progress_manager = None
self.download_manager.blob_handler = None
self.download_manager.wallet_info_exchanger = None
self.blob_requester = None
self.download_manager.connection_manager = None
self.download_manager = None
def _get_primary_request_creators(self, download_manager):
return [self.blob_requester]
def _get_secondary_request_creators(self, download_manager):
return [download_manager.wallet_info_exchanger]
def _get_metadata_handler(self, download_manager):
pass
def _get_blob_requester(self, download_manager):
return BlobRequester(self.blob_manager, self.peer_finder,
self.payment_rate_manager, self.wallet,
download_manager)
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading,
self.blob_manager, download_manager)
def _get_write_func(self):
pass
def _get_blob_handler(self, download_manager):
return CryptBlobHandler(self.key, self._get_write_func())
def _get_connection_manager(self, download_manager):
return ConnectionManager(self, self.rate_limiter,
self._get_primary_request_creators(download_manager),
self._get_secondary_request_creators(download_manager))
def _fire_completed_deferred(self, err=None):
self.finished_deferred, d = None, self.finished_deferred
if d is not None:
if err is not None:
d.errback(err)
else:
value = self._get_finished_deferred_callback_value()
d.callback(value)
else:
log.debug("Not firing the completed deferred because d is None")
def _get_finished_deferred_callback_value(self):
return None
def _finished_downloading(self, finished):
if finished is True:
self.completed = True
return self.stop()
def insufficient_funds(self, err):
return self.stop(err=err)
@@ -0,0 +1,199 @@
import os
import logging
import traceback
from binascii import hexlify, unhexlify
from twisted.internet import defer, threads
from lbrynet.conf import Config
from lbrynet.extras.compat import f2d
from lbrynet.p2p.StreamDescriptor import save_sd_info
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.p2p.Error import FileOpenError
from lbrynet.blob.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
log = logging.getLogger(__name__)
class EncryptedFileDownloader(CryptStreamDownloader):
"""Classes which inherit from this class download LBRY files"""
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager,
storage, payment_rate_manager, wallet, key, stream_name, file_name):
super().__init__(conf, peer_finder, rate_limiter, blob_manager,
payment_rate_manager, wallet, key, stream_name)
self.stream_hash = stream_hash
self.storage = storage
self.file_name = os.path.basename(unhexlify(file_name).decode())
self._calculated_total_bytes = None
@defer.inlineCallbacks
def delete_data(self):
crypt_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
blob_hashes = [b.blob_hash for b in crypt_infos if b.blob_hash]
sd_hash = yield f2d(self.storage.get_sd_blob_hash_for_stream(self.stream_hash))
blob_hashes.append(sd_hash)
yield self.blob_manager.delete_blobs(blob_hashes)
def stop(self, err=None):
self._close_output()
return super().stop(err=err)
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading,
self.blob_manager, download_manager)
def _start(self):
def check_start_succeeded(success):
if success:
self.starting = False
self.stopped = False
self.completed = False
return True
else:
return self._start_failed()
self.download_manager = self._get_download_manager()
d = self._setup_output()
d.addCallback(lambda _: self.download_manager.start_downloading())
d.addCallbacks(check_start_succeeded)
return d
def _setup_output(self):
pass
def _close_output(self):
pass
async def get_total_bytes(self):
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
return sum([b.length for b in blobs])
def get_total_bytes_cached(self):
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
if self.download_manager is None:
return 0
else:
self._calculated_total_bytes = self.download_manager.calculate_total_bytes()
return self._calculated_total_bytes
def get_bytes_left_to_output(self):
if self.download_manager is not None:
return self.download_manager.calculate_bytes_left_to_output()
else:
return 0
def get_bytes_left_to_download(self):
if self.download_manager is not None:
return self.download_manager.calculate_bytes_left_to_download()
else:
return 0
def _get_metadata_handler(self, download_manager):
return EncryptedFileMetadataHandler(self.stream_hash,
self.storage, download_manager)
class EncryptedFileDownloaderFactory:
#implements(IStreamDownloaderFactory)
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet):
self.conf = conf
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.blob_manager = blob_manager
self.storage = storage
self.wallet = wallet
def can_download(self, sd_validator):
return True
def make_downloader(self, metadata, options, payment_rate_manager, **kwargs):
assert len(options) == 1
data_rate = options[0]
payment_rate_manager.min_blob_data_payment_rate = data_rate
def save_source_if_blob(stream_hash):
return defer.succeed(metadata.source_blob_hash)
def create_downloader(stream_hash):
downloader = self._make_downloader(stream_hash, payment_rate_manager,
metadata.validator.raw_info)
return defer.succeed(downloader)
d = save_sd_info(self.blob_manager, metadata.source_blob_hash, metadata.validator.raw_info)
d.addCallback(save_source_if_blob)
d.addCallback(create_downloader)
return d
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
pass
class EncryptedFileSaver(EncryptedFileDownloader):
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
payment_rate_manager, wallet, download_directory, key, stream_name, file_name):
super().__init__(conf, stream_hash, peer_finder, rate_limiter,
blob_manager, storage, payment_rate_manager,
wallet, key, stream_name, file_name)
self.download_directory = unhexlify(download_directory).decode()
self.file_written_to = os.path.join(self.download_directory, unhexlify(file_name).decode())
self.file_handle = None
def __str__(self):
return str(self.file_written_to)
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading,
self.blob_manager,
download_manager)
def _setup_output(self):
def open_file():
if self.file_handle is None:
file_written_to = os.path.join(self.download_directory, self.file_name)
try:
self.file_handle = open(file_written_to, 'wb')
self.file_written_to = file_written_to
except IOError:
log.error(traceback.format_exc())
raise FileOpenError(
"Failed to open %s. Make sure you have permission to save files to that"
" location." % file_written_to
)
return threads.deferToThread(open_file)
def _close_output(self):
self.file_handle, file_handle = None, self.file_handle
if file_handle is not None:
name = file_handle.name
file_handle.close()
if self.completed is False:
os.remove(name)
def _get_write_func(self):
def write_func(data):
if self.stopped is False and self.file_handle is not None:
self.file_handle.write(data)
return write_func
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
super().__init__(conf, peer_finder, rate_limiter, blob_manager, storage, wallet)
self.download_directory = hexlify(download_directory.encode())
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
stream_name = stream_info.raw_info['stream_name']
key = stream_info.raw_info['key']
suggested_file_name = stream_info.raw_info['suggested_file_name']
return EncryptedFileSaver(
self.conf, stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage,
payment_rate_manager, self.wallet, self.download_directory, key=key, stream_name=stream_name,
file_name=suggested_file_name
)
@staticmethod
def get_description():
return "Save"
@@ -0,0 +1,40 @@
import logging
from twisted.internet import defer
from lbrynet.extras.compat import f2d
log = logging.getLogger(__name__)
class EncryptedFileMetadataHandler:
def __init__(self, stream_hash, storage, download_manager):
self.stream_hash = stream_hash
self.storage = storage
self.download_manager = download_manager
self._final_blob_num = None
######### IMetadataHandler #########
@defer.inlineCallbacks
def get_initial_blobs(self):
blob_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
return self._format_initial_blobs_for_download_manager(blob_infos)
def final_blob_num(self):
return self._final_blob_num
######### internal calls #########
def _format_initial_blobs_for_download_manager(self, blob_infos):
infos = []
for i, crypt_blob in enumerate(blob_infos):
if crypt_blob.blob_hash is not None and crypt_blob.length:
infos.append(crypt_blob)
else:
if i != len(blob_infos) - 1:
raise Exception("Invalid stream terminator: %i of %i" %
(i, len(blob_infos) - 1))
log.debug("Setting _final_blob_num to %s", str(crypt_blob.blob_num - 1))
self._final_blob_num = crypt_blob.blob_num - 1
return infos
@@ -0,0 +1,47 @@
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamDescriptorValidator
from lbrynet.p2p.DownloadOption import DownloadOption, DownloadOptionChoice
def add_lbry_file_to_sd_identifier(sd_identifier):
sd_identifier.add_stream_type(EncryptedFileStreamType, EncryptedFileStreamDescriptorValidator,
EncryptedFileOptions())
class EncryptedFileOptions:
def __init__(self):
pass
def get_downloader_options(self, sd_validator, payment_rate_manager):
prm = payment_rate_manager
def get_default_data_rate_description():
if prm.base.min_blob_data_payment_rate is None:
return "Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)
else:
return "%f LBC/MB" % prm.base.min_blob_data_payment_rate
rate_choices = []
rate_choices.append(DownloadOptionChoice(
prm.base.min_blob_data_payment_rate,
"No change - %s" % get_default_data_rate_description(),
"No change - %s" % get_default_data_rate_description()))
if prm.base.min_blob_data_payment_rate is not None:
rate_choices.append(DownloadOptionChoice(
None,
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate),
"Application default (%s LBC/MB)" % str(prm.base.min_blob_data_payment_rate)))
rate_choices.append(DownloadOptionChoice(float,
"Enter rate in LBC/MB",
"Enter rate in LBC/MB"))
options = [
DownloadOption(
rate_choices,
"Rate which will be paid for data",
"data payment rate",
prm.base.min_blob_data_payment_rate,
get_default_data_rate_description()
),
]
return options
File renamed without changes.
+51
View File
@@ -0,0 +1,51 @@
import os
import logging
from io import BytesIO
from twisted.internet import defer
from twisted.web.client import FileBodyProducer
from lbrynet.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
class BlobFileCreator:
"""
This class is used to create blobs on the local filesystem
when we do not know the blob hash beforehand (i.e, when creating
a new stream)
"""
def __init__(self, blob_dir):
self.blob_dir = blob_dir
self.buffer = BytesIO()
self._is_open = True
self._hashsum = get_lbry_hash_obj()
self.len_so_far = 0
self.blob_hash = None
self.length = None
@defer.inlineCallbacks
def close(self):
self.length = self.len_so_far
self.blob_hash = self._hashsum.hexdigest()
if self.blob_hash and self._is_open and self.length > 0:
# do not save 0 length files (empty tail blob in streams)
# or if its been closed already
self.buffer.seek(0)
out_path = os.path.join(self.blob_dir, self.blob_hash)
producer = FileBodyProducer(self.buffer)
yield producer.startProducing(open(out_path, 'wb'))
self._is_open = False
if self.length > 0:
defer.returnValue(self.blob_hash)
else:
# 0 length files (empty tail blob in streams )
# must return None as their blob_hash for
# it to be saved properly by EncryptedFileMetadataManagers
defer.returnValue(None)
def write(self, data):
if not self._is_open:
raise IOError
self._hashsum.update(data)
self.len_so_far += len(data)
self.buffer.write(data)
+30
View File
@@ -0,0 +1,30 @@
import logging
log = logging.getLogger(__name__)
class HashBlobReader:
"""
This is a file like reader class that supports
read(size) and close()
"""
def __init__(self, read_handle, finished_cb):
self.finished_cb = finished_cb
self.finished_cb_d = None
self.read_handle = read_handle
def __del__(self):
if self.finished_cb_d is None:
log.warning("Garbage collection was called, but reader for %s was not closed yet",
self.read_handle.name)
self.close()
def read(self, size=-1):
return self.read_handle.read(size)
def close(self):
# if we've already closed and called finished_cb, do nothing
if self.finished_cb_d is not None:
return
self.read_handle.close()
self.finished_cb_d = self.finished_cb(self)
+35 -45
View File
@@ -1,68 +1,58 @@
import typing
import logging
import asyncio
from io import BytesIO
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from twisted.python.failure import Failure
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
from lbrynet.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
class HashBlobWriter:
def __init__(self, expected_blob_hash: str, get_length: typing.Callable[[], int],
finished: asyncio.Future):
self.expected_blob_hash = expected_blob_hash
self.get_length = get_length
self.buffer = BytesIO()
self.finished = finished
self.finished.add_done_callback(lambda *_: self.close_handle())
def __init__(self, length_getter, finished_cb):
self.write_handle = BytesIO()
self.length_getter = length_getter
self.finished_cb = finished_cb
self.finished_cb_d = None
self._hashsum = get_lbry_hash_obj()
self.len_so_far = 0
def __del__(self):
if self.buffer is not None:
if self.finished_cb_d is None:
log.warning("Garbage collection was called, but writer was not closed yet")
self.close_handle()
self.close()
def calculate_blob_hash(self) -> str:
@property
def blob_hash(self):
return self._hashsum.hexdigest()
def closed(self):
return self.buffer is None or self.buffer.closed
def write(self, data: bytes):
expected_length = self.get_length()
if not expected_length:
raise IOError("unknown blob length")
if self.buffer is None:
def write(self, data):
if self.write_handle is None:
log.warning("writer has already been closed")
if not self.finished.done():
self.finished.cancel()
return
raise IOError('I/O operation on closed file')
self._hashsum.update(data)
self.len_so_far += len(data)
if self.len_so_far > expected_length:
self.finished.set_exception(InvalidDataError(
f'Length so far is greater than the expected length. {self.len_so_far} to {expected_length}'
))
self.close_handle()
return
self.buffer.write(data)
if self.len_so_far == expected_length:
blob_hash = self.calculate_blob_hash()
if blob_hash != self.expected_blob_hash:
self.finished.set_exception(InvalidBlobHashError(
f"blob hash is {blob_hash} vs expected {self.expected_blob_hash}"
))
elif self.finished and not (self.finished.done() or self.finished.cancelled()):
self.finished.set_result(self.buffer.getvalue())
self.close_handle()
if self.len_so_far > self.length_getter():
self.finished_cb_d = self.finished_cb(
self,
Failure(InvalidDataError("Length so far is greater than the expected length."
" %s to %s" % (self.len_so_far,
self.length_getter()))))
else:
self.write_handle.write(data)
if self.len_so_far == self.length_getter():
self.finished_cb_d = self.finished_cb(self)
def close_handle(self):
if not self.finished.done():
self.finished.cancel()
if self.buffer is not None:
self.buffer.close()
self.buffer = None
if self.write_handle is not None:
self.write_handle.close()
self.write_handle = None
def close(self, reason=None):
# if we've already called finished_cb because we either finished writing
# or closed already, do nothing
if self.finished_cb_d is not None:
return
if reason is None:
reason = Failure(DownloadCanceledError())
self.finished_cb_d = self.finished_cb(self, reason)
-245
View File
@@ -1,245 +0,0 @@
import asyncio
import logging
import typing
import binascii
from lbrynet.error import InvalidBlobHashError, InvalidDataError
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest
from lbrynet.utils import cache_concurrent
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_file import AbstractBlob
from lbrynet.blob.writer import HashBlobWriter
from lbrynet.connection_manager import ConnectionManager
log = logging.getLogger(__name__)
class BlobExchangeClientProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, peer_timeout: typing.Optional[float] = 10,
connection_manager: typing.Optional['ConnectionManager'] = None):
self.loop = loop
self.peer_port: typing.Optional[int] = None
self.peer_address: typing.Optional[str] = None
self.transport: typing.Optional[asyncio.Transport] = None
self.peer_timeout = peer_timeout
self.connection_manager = connection_manager
self.writer: typing.Optional['HashBlobWriter'] = None
self.blob: typing.Optional['AbstractBlob'] = None
self._blob_bytes_received = 0
self._response_fut: typing.Optional[asyncio.Future] = None
self.buf = b''
# this is here to handle the race when the downloader is closed right as response_fut gets a result
self.closed = asyncio.Event(loop=self.loop)
def data_received(self, data: bytes):
if self.connection_manager:
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
# assert self.peer_address is not None
self.connection_manager.received_data(f"{self.peer_address}:{self.peer_port}", len(data))
#log.debug("%s:%d -- got %s bytes -- %s bytes on buffer -- %s blob bytes received",
# self.peer_address, self.peer_port, len(data), len(self.buf), self._blob_bytes_received)
if not self.transport or self.transport.is_closing():
log.warning("transport closing, but got more bytes from %s:%i\n%s", self.peer_address, self.peer_port,
binascii.hexlify(data))
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
return
if not self._response_fut:
log.warning("Protocol received data before expected, probable race on keep alive. Closing transport.")
return self.close()
if self._blob_bytes_received and not self.writer.closed():
return self._write(data)
response = BlobResponse.deserialize(self.buf + data)
if not response.responses and not self._response_fut.done():
self.buf += data
return
else:
self.buf = b''
if response.responses and self.blob:
blob_response = response.get_blob_response()
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
# set the expected length for the incoming blob if we didn't know it
self.blob.set_length(blob_response.length)
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
# the server started sending a blob we didn't request
log.warning("mismatch with self.blob %s", self.blob.blob_hash)
return
if response.responses:
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
# fire the Future with the response to our request
self._response_fut.set_result(response)
if response.blob_data and self.writer and not self.writer.closed():
# log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
# write blob bytes if we're writing a blob and have blob bytes to write
self._write(response.blob_data)
def _write(self, data: bytes):
if len(data) > (self.blob.get_length() - self._blob_bytes_received):
data = data[:(self.blob.get_length() - self._blob_bytes_received)]
log.warning("got more than asked from %s:%d, probable sendfile bug", self.peer_address, self.peer_port)
else:
data = data
self._blob_bytes_received += len(data)
try:
self.writer.write(data)
except IOError as err:
log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
except (asyncio.TimeoutError) as err: # TODO: is this needed?
log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err)
async def _download_blob(self) -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
:return: download success (bool), keep connection (bool)
"""
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
blob_hash = self.blob.blob_hash
if not self.peer_address:
addr_info = self.transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr_info
try:
msg = request.serialize()
log.debug("send request to %s:%i -> %s", self.peer_address, self.peer_port, msg.decode())
self.transport.write(msg)
if self.connection_manager:
self.connection_manager.sent_data(f"{self.peer_address}:{self.peer_port}", len(msg))
response: BlobResponse = await asyncio.wait_for(self._response_fut, self.peer_timeout, loop=self.loop)
availability_response = response.get_availability_response()
price_response = response.get_price_response()
blob_response = response.get_blob_response()
if self.closed.is_set():
msg = f"cancelled blob request for {blob_hash} immediately after we got a response"
log.warning(msg)
raise asyncio.CancelledError(msg)
if (not blob_response or blob_response.error) and\
(not availability_response or not availability_response.available_blobs):
log.warning("%s not in availability response from %s:%i", self.blob.blob_hash, self.peer_address,
self.peer_port)
log.warning(response.to_dict())
return self._blob_bytes_received, self.close()
elif availability_response.available_blobs and \
availability_response.available_blobs != [self.blob.blob_hash]:
log.warning("blob availability response doesn't match our request from %s:%i",
self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not price_response or price_response.blob_data_payment_rate != 'RATE_ACCEPTED':
log.warning("data rate rejected by %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response or blob_response.error:
log.warning("blob cant be downloaded from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if not blob_response.error and blob_response.blob_hash != self.blob.blob_hash:
log.warning("incoming blob hash mismatch from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
if self.blob.length is not None and self.blob.length != blob_response.length:
log.warning("incoming blob unexpected length from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
msg = f"downloading {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}," \
f" timeout in {self.peer_timeout}"
log.debug(msg)
msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}"
await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop)
log.info(msg)
# await self.blob.finished_writing.wait() not necessary, but a dangerous change. TODO: is it needed?
return self._blob_bytes_received, self.transport
except asyncio.TimeoutError:
return self._blob_bytes_received, self.close()
except (InvalidBlobHashError, InvalidDataError):
log.warning("invalid blob from %s:%i", self.peer_address, self.peer_port)
return self._blob_bytes_received, self.close()
def close(self):
self.closed.set()
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
if self.writer and not self.writer.closed():
self.writer.close_handle()
self._response_fut = None
self.writer = None
self.blob = None
if self.transport:
self.transport.close()
self.transport = None
self.buf = b''
async def download_blob(self, blob: 'AbstractBlob') -> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
self.closed.clear()
blob_hash = blob.blob_hash
if blob.get_is_verified() or not blob.is_writeable():
return 0, self.transport
try:
self._blob_bytes_received = 0
self.blob, self.writer = blob, blob.get_blob_writer(self.peer_address, self.peer_port)
self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob()
except OSError as e:
# i'm not sure how to fix this race condition - jack
log.warning("race happened downloading %s from %s:%i", blob_hash, self.peer_address, self.peer_port)
# return self._blob_bytes_received, self.transport
raise
except asyncio.TimeoutError:
if self._response_fut and not self._response_fut.done():
self._response_fut.cancel()
self.close()
return self._blob_bytes_received, None
except asyncio.CancelledError:
self.close()
raise
finally:
if self.writer and not self.writer.closed():
self.writer.close_handle()
self.writer = None
def connection_made(self, transport: asyncio.Transport):
addr = transport.get_extra_info('peername')
self.peer_address, self.peer_port = addr[0], addr[1]
self.transport = transport
if self.connection_manager:
self.connection_manager.connection_made(f"{self.peer_address}:{self.peer_port}")
log.debug("connection made to %s:%i", self.peer_address, self.peer_port)
def connection_lost(self, reason):
if self.connection_manager:
self.connection_manager.outgoing_connection_lost(f"{self.peer_address}:{self.peer_port}")
log.debug("connection lost to %s:%i (reason: %s, %s)", self.peer_address, self.peer_port, str(reason),
str(type(reason)))
self.close()
@cache_concurrent
async def request_blob(loop: asyncio.BaseEventLoop, blob: 'AbstractBlob', address: str, tcp_port: int,
peer_connect_timeout: float, blob_download_timeout: float,
connected_transport: asyncio.Transport = None, connection_id: int = 0,
connection_manager: typing.Optional['ConnectionManager'] = None)\
-> typing.Tuple[int, typing.Optional[asyncio.Transport]]:
"""
Returns [<downloaded blob>, <keep connection>]
"""
protocol = BlobExchangeClientProtocol(
loop, blob_download_timeout, connection_manager
)
if connected_transport and not connected_transport.is_closing():
connected_transport.set_protocol(protocol)
protocol.transport = connected_transport
log.debug("reusing connection for %s:%d", address, tcp_port)
else:
connected_transport = None
try:
if not connected_transport:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop)
if blob.get_is_verified() or not blob.is_writeable():
# file exists but not verified means someone is writing right now, give it time, come back later
return 0, connected_transport
return await protocol.download_blob(blob)
except (asyncio.TimeoutError, ConnectionRefusedError, ConnectionAbortedError, OSError):
return 0, None
-130
View File
@@ -1,130 +0,0 @@
import asyncio
import typing
import logging
from lbrynet.utils import cache_concurrent
from lbrynet.blob_exchange.client import request_blob
if typing.TYPE_CHECKING:
from lbrynet.conf import Config
from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob.blob_file import AbstractBlob
log = logging.getLogger(__name__)
class BlobDownloader:
BAN_FACTOR = 2.0 # fixme: when connection manager gets implemented, move it out from here
def __init__(self, loop: asyncio.BaseEventLoop, config: 'Config', blob_manager: 'BlobManager',
peer_queue: asyncio.Queue):
self.loop = loop
self.config = config
self.blob_manager = blob_manager
self.peer_queue = peer_queue
self.active_connections: typing.Dict['KademliaPeer', asyncio.Task] = {} # active request_blob calls
self.ignored: typing.Dict['KademliaPeer', int] = {}
self.scores: typing.Dict['KademliaPeer', int] = {}
self.failures: typing.Dict['KademliaPeer', int] = {}
self.connections: typing.Dict['KademliaPeer', asyncio.Transport] = {}
self.is_running = asyncio.Event(loop=self.loop)
def should_race_continue(self, blob: 'AbstractBlob'):
if len(self.active_connections) >= self.config.max_connections_per_download:
return False
return not (blob.get_is_verified() or not blob.is_writeable())
async def request_blob_from_peer(self, blob: 'AbstractBlob', peer: 'KademliaPeer', connection_id: int = 0):
if blob.get_is_verified():
return
transport = self.connections.get(peer)
start = self.loop.time()
bytes_received, transport = await request_blob(
self.loop, blob, peer.address, peer.tcp_port, self.config.peer_connect_timeout,
self.config.blob_download_timeout, connected_transport=transport, connection_id=connection_id,
connection_manager=self.blob_manager.connection_manager
)
if not transport and peer not in self.ignored:
self.ignored[peer] = self.loop.time()
log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
self.failures[peer] = self.failures.get(peer, 0) + 1
if peer in self.connections:
del self.connections[peer]
elif transport:
log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
self.failures[peer] = 0
self.connections[peer] = transport
elapsed = self.loop.time() - start
self.scores[peer] = bytes_received / elapsed if bytes_received and elapsed else 1
async def new_peer_or_finished(self):
active_tasks = list(self.active_connections.values()) + [asyncio.sleep(1)]
await asyncio.wait(active_tasks, loop=self.loop, return_when='FIRST_COMPLETED')
def cleanup_active(self):
if not self.active_connections and not self.connections:
self.clearbanned()
to_remove = [peer for (peer, task) in self.active_connections.items() if task.done()]
for peer in to_remove:
del self.active_connections[peer]
def clearbanned(self):
now = self.loop.time()
self.ignored = dict((
(peer, when) for (peer, when) in self.ignored.items()
if (now - when) < min(30.0, (self.failures.get(peer, 0) ** self.BAN_FACTOR))
))
@cache_concurrent
async def download_blob(self, blob_hash: str, length: typing.Optional[int] = None,
connection_id: int = 0) -> 'AbstractBlob':
blob = self.blob_manager.get_blob(blob_hash, length)
if blob.get_is_verified():
return blob
self.is_running.set()
try:
while not blob.get_is_verified() and self.is_running.is_set():
batch: typing.Set['KademliaPeer'] = set()
while not self.peer_queue.empty():
batch.update(self.peer_queue.get_nowait())
if batch:
self.peer_queue.put_nowait(list(batch))
log.debug(
"running, %d peers, %d ignored, %d active",
len(batch), len(self.ignored), len(self.active_connections)
)
for peer in sorted(batch, key=lambda peer: self.scores.get(peer, 0), reverse=True):
if not self.should_race_continue(blob):
break
if peer not in self.active_connections and peer not in self.ignored:
log.debug("request %s from %s:%i", blob_hash[:8], peer.address, peer.tcp_port)
t = self.loop.create_task(self.request_blob_from_peer(blob, peer, connection_id))
self.active_connections[peer] = t
await self.new_peer_or_finished()
self.cleanup_active()
log.debug("downloaded %s", blob_hash[:8])
return blob
finally:
blob.close()
def close(self):
self.scores.clear()
self.ignored.clear()
self.is_running.clear()
for transport in self.connections.values():
transport.close()
async def download_blob(loop, config: 'Config', blob_manager: 'BlobManager', node: 'Node',
blob_hash: str) -> 'AbstractBlob':
search_queue = asyncio.Queue(loop=loop, maxsize=config.max_connections_per_download)
search_queue.put_nowait(blob_hash)
peer_queue, accumulate_task = node.accumulate_peers(search_queue)
downloader = BlobDownloader(loop, config, blob_manager, peer_queue)
try:
return await downloader.download_blob(blob_hash)
finally:
if accumulate_task and not accumulate_task.done():
accumulate_task.cancel()
downloader.close()
-283
View File
@@ -1,283 +0,0 @@
import typing
import json
import logging
log = logging.getLogger(__name__)
class BlobMessage:
key = ''
def to_dict(self) -> typing.Dict:
raise NotImplementedError()
class BlobPriceRequest(BlobMessage):
key = 'blob_data_payment_rate'
def __init__(self, blob_data_payment_rate: float, **kwargs) -> None:
self.blob_data_payment_rate = blob_data_payment_rate
def to_dict(self) -> typing.Dict:
return {
self.key: self.blob_data_payment_rate
}
class BlobPriceResponse(BlobMessage):
key = 'blob_data_payment_rate'
rate_accepted = 'RATE_ACCEPTED'
rate_too_low = 'RATE_TOO_LOW'
rate_unset = 'RATE_UNSET'
def __init__(self, blob_data_payment_rate: str, **kwargs) -> None:
if blob_data_payment_rate not in (self.rate_accepted, self.rate_too_low, self.rate_unset):
raise ValueError(blob_data_payment_rate)
self.blob_data_payment_rate = blob_data_payment_rate
def to_dict(self) -> typing.Dict:
return {
self.key: self.blob_data_payment_rate
}
class BlobAvailabilityRequest(BlobMessage):
key = 'requested_blobs'
def __init__(self, requested_blobs: typing.List[str], lbrycrd_address: typing.Optional[bool] = True,
**kwargs) -> None:
assert len(requested_blobs)
self.requested_blobs = requested_blobs
self.lbrycrd_address = lbrycrd_address
def to_dict(self) -> typing.Dict:
return {
self.key: self.requested_blobs,
'lbrycrd_address': self.lbrycrd_address
}
class BlobAvailabilityResponse(BlobMessage):
key = 'available_blobs'
def __init__(self, available_blobs: typing.List[str], lbrycrd_address: typing.Optional[str] = True,
**kwargs) -> None:
self.available_blobs = available_blobs
self.lbrycrd_address = lbrycrd_address
def to_dict(self) -> typing.Dict:
d = {
self.key: self.available_blobs
}
if self.lbrycrd_address:
d['lbrycrd_address'] = self.lbrycrd_address
return d
class BlobDownloadRequest(BlobMessage):
key = 'requested_blob'
def __init__(self, requested_blob: str, **kwargs) -> None:
self.requested_blob = requested_blob
def to_dict(self) -> typing.Dict:
return {
self.key: self.requested_blob
}
class BlobDownloadResponse(BlobMessage):
key = 'incoming_blob'
def __init__(self, **response: typing.Dict) -> None:
incoming_blob = response[self.key]
self.error = None
self.incoming_blob = None
if 'error' in incoming_blob:
self.error = incoming_blob['error']
else:
self.incoming_blob = {'blob_hash': incoming_blob['blob_hash'], 'length': incoming_blob['length']}
self.length = None if not self.incoming_blob else self.incoming_blob['length']
self.blob_hash = None if not self.incoming_blob else self.incoming_blob['blob_hash']
def to_dict(self) -> typing.Dict:
return {
self.key if not self.error else 'error': self.incoming_blob or self.error,
}
class BlobPaymentAddressRequest(BlobMessage):
key = 'lbrycrd_address'
def __init__(self, lbrycrd_address: str, **kwargs) -> None:
self.lbrycrd_address = lbrycrd_address
def to_dict(self) -> typing.Dict:
return {
self.key: self.lbrycrd_address
}
class BlobPaymentAddressResponse(BlobPaymentAddressRequest):
pass
class BlobErrorResponse(BlobMessage):
key = 'error'
def __init__(self, error: str, **kwargs) -> None:
self.error = error
def to_dict(self) -> typing.Dict:
return {
self.key: self.error
}
blob_request_types = typing.Union[BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
BlobPaymentAddressRequest]
blob_response_types = typing.Union[BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
BlobErrorResponse, BlobPaymentAddressResponse]
def _parse_blob_response(response_msg: bytes) -> typing.Tuple[typing.Optional[typing.Dict], bytes]:
# scenarios:
# <json>
# <blob bytes>
# <json><blob bytes>
curr_pos = 0
while True:
next_close_paren = response_msg.find(b'}', curr_pos)
if next_close_paren == -1:
return None, response_msg
curr_pos = next_close_paren + 1
try:
response = json.loads(response_msg[:curr_pos])
except ValueError:
continue
possible_response_keys = {
BlobPaymentAddressResponse.key,
BlobAvailabilityResponse.key,
BlobPriceResponse.key,
BlobDownloadResponse.key
}
if isinstance(response, dict) and response.keys():
if set(response.keys()).issubset(possible_response_keys):
return response, response_msg[curr_pos:]
return None, response_msg
class BlobRequest:
def __init__(self, requests: typing.List[blob_request_types]) -> None:
self.requests = requests
def to_dict(self):
d = {}
for request in self.requests:
d.update(request.to_dict())
return d
def _get_request(self, request_type: blob_request_types):
request = tuple(filter(lambda r: type(r) == request_type, self.requests))
if request:
return request[0]
def get_availability_request(self) -> typing.Optional[BlobAvailabilityRequest]:
response = self._get_request(BlobAvailabilityRequest)
if response:
return response
def get_price_request(self) -> typing.Optional[BlobPriceRequest]:
response = self._get_request(BlobPriceRequest)
if response:
return response
def get_blob_request(self) -> typing.Optional[BlobDownloadRequest]:
response = self._get_request(BlobDownloadRequest)
if response:
return response
def get_address_request(self) -> typing.Optional[BlobPaymentAddressRequest]:
response = self._get_request(BlobPaymentAddressRequest)
if response:
return response
def serialize(self) -> bytes:
return json.dumps(self.to_dict()).encode()
@classmethod
def deserialize(cls, data: bytes) -> 'BlobRequest':
request = json.loads(data)
return cls([
request_type(**request)
for request_type in (BlobPriceRequest, BlobAvailabilityRequest, BlobDownloadRequest,
BlobPaymentAddressRequest)
if request_type.key in request
])
@classmethod
def make_request_for_blob_hash(cls, blob_hash: str) -> 'BlobRequest':
return cls(
[BlobAvailabilityRequest([blob_hash]), BlobPriceRequest(0.0), BlobDownloadRequest(blob_hash)]
)
class BlobResponse:
def __init__(self, responses: typing.List[blob_response_types], blob_data: typing.Optional[bytes] = None) -> None:
self.responses = responses
self.blob_data = blob_data
def to_dict(self):
d = {}
for response in self.responses:
d.update(response.to_dict())
return d
def _get_response(self, response_type: blob_response_types):
response = tuple(filter(lambda r: type(r) == response_type, self.responses))
if response:
return response[0]
def get_error_response(self) -> typing.Optional[BlobErrorResponse]:
error = self._get_response(BlobErrorResponse)
if error:
log.error(error)
return error
def get_availability_response(self) -> typing.Optional[BlobAvailabilityResponse]:
response = self._get_response(BlobAvailabilityResponse)
if response:
return response
def get_price_response(self) -> typing.Optional[BlobPriceResponse]:
response = self._get_response(BlobPriceResponse)
if response:
return response
def get_blob_response(self) -> typing.Optional[BlobDownloadResponse]:
response = self._get_response(BlobDownloadResponse)
if response:
return response
def get_address_response(self) -> typing.Optional[BlobPaymentAddressResponse]:
response = self._get_response(BlobPaymentAddressResponse)
if response:
return response
def serialize(self) -> bytes:
return json.dumps(self.to_dict()).encode()
@classmethod
def deserialize(cls, data: bytes) -> 'BlobResponse':
response, extra = _parse_blob_response(data)
requests = []
if response:
requests.extend([
response_type(**response)
for response_type in (BlobPriceResponse, BlobAvailabilityResponse, BlobDownloadResponse,
BlobErrorResponse, BlobPaymentAddressResponse)
if response_type.key in response
])
return cls(requests, extra)
-134
View File
@@ -1,134 +0,0 @@
import asyncio
import binascii
import logging
import typing
from json.decoder import JSONDecodeError
from lbrynet.blob_exchange.serialization import BlobResponse, BlobRequest, blob_response_types
from lbrynet.blob_exchange.serialization import BlobAvailabilityResponse, BlobPriceResponse, BlobDownloadResponse, \
BlobPaymentAddressResponse
if typing.TYPE_CHECKING:
from lbrynet.blob.blob_manager import BlobManager
log = logging.getLogger(__name__)
class BlobServerProtocol(asyncio.Protocol):
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: asyncio.Task = None
self.started_listening = asyncio.Event(loop=self.loop)
self.buf = b''
self.transport = None
self.lbrycrd_address = lbrycrd_address
self.peer_address_and_port: typing.Optional[str] = None
def connection_made(self, transport):
self.transport = transport
self.peer_address_and_port = "%s:%i" % self.transport.get_extra_info('peername')
self.blob_manager.connection_manager.connection_received(self.peer_address_and_port)
def connection_lost(self, exc: typing.Optional[Exception]) -> None:
self.blob_manager.connection_manager.incoming_connection_lost(self.peer_address_and_port)
def send_response(self, responses: typing.List[blob_response_types]):
to_send = []
while responses:
to_send.append(responses.pop())
serialized = BlobResponse(to_send).serialize()
self.transport.write(serialized)
self.blob_manager.connection_manager.sent_data(self.peer_address_and_port, len(serialized))
async def handle_request(self, request: BlobRequest):
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
responses = []
address_request = request.get_address_request()
if address_request:
responses.append(BlobPaymentAddressResponse(lbrycrd_address=self.lbrycrd_address))
availability_request = request.get_availability_request()
if availability_request:
responses.append(BlobAvailabilityResponse(available_blobs=list(set((
filter(lambda blob_hash: blob_hash in self.blob_manager.completed_blob_hashes,
availability_request.requested_blobs)
)))))
price_request = request.get_price_request()
if price_request:
responses.append(BlobPriceResponse(blob_data_payment_rate='RATE_ACCEPTED'))
download_request = request.get_blob_request()
if download_request:
blob = self.blob_manager.get_blob(download_request.requested_blob)
if blob.get_is_verified():
incoming_blob = {'blob_hash': blob.blob_hash, 'length': blob.length}
responses.append(BlobDownloadResponse(incoming_blob=incoming_blob))
self.send_response(responses)
log.debug("send %s to %s:%i", blob.blob_hash[:8], peer_address, peer_port)
try:
sent = await blob.sendfile(self)
except (ConnectionResetError, BrokenPipeError, RuntimeError, OSError):
if self.transport:
self.transport.close()
return
log.info("sent %s (%i bytes) to %s:%i", blob.blob_hash[:8], sent, peer_address, peer_port)
if responses:
self.send_response(responses)
# self.transport.close()
def data_received(self, data):
request = None
if data:
self.blob_manager.connection_manager.received_data(self.peer_address_and_port, len(data))
message, separator, remainder = data.rpartition(b'}')
if not separator:
self.buf += data
return
try:
request = BlobRequest.deserialize(self.buf + data)
self.buf = remainder
except JSONDecodeError:
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
log.error("failed to decode blob request from %s:%i (%i bytes): %s", peer_address, peer_port,
len(data), '' if not data else binascii.hexlify(data).decode())
if not request:
addr = self.transport.get_extra_info('peername')
peer_address, peer_port = addr
log.warning("failed to decode blob request from %s:%i", peer_address, peer_port)
self.transport.close()
return
self.loop.create_task(self.handle_request(request))
class BlobServer:
def __init__(self, loop: asyncio.BaseEventLoop, blob_manager: 'BlobManager', lbrycrd_address: str):
self.loop = loop
self.blob_manager = blob_manager
self.server_task: typing.Optional[asyncio.Task] = None
self.started_listening = asyncio.Event(loop=self.loop)
self.lbrycrd_address = lbrycrd_address
self.server_protocol_class = BlobServerProtocol
def start_server(self, port: int, interface: typing.Optional[str] = '0.0.0.0'):
if self.server_task is not None:
raise Exception("already running")
async def _start_server():
server = await self.loop.create_server(
lambda: self.server_protocol_class(self.loop, self.blob_manager, self.lbrycrd_address),
interface, port
)
self.started_listening.set()
log.info("Blob server listening on TCP %s:%i", interface, port)
async with server:
await server.serve_forever()
self.server_task = self.loop.create_task(_start_server())
def stop_server(self):
if self.server_task:
self.server_task.cancel()
self.server_task = None
log.info("Stopped blob server")
-1
View File
@@ -1,3 +1,2 @@
# don't touch this. CI server changes this during build/deployment
BUILD = "dev"
BUILD_COMMIT = "source installation"
+111 -223
View File
@@ -2,13 +2,13 @@ import os
import re
import sys
import typing
import json
import logging
import yaml
from argparse import ArgumentParser
from contextlib import contextmanager
from appdirs import user_data_dir, user_config_dir
from lbrynet.error import InvalidCurrencyError
from lbrynet.dht import constants
from lbrynet.p2p.Error import InvalidCurrencyError
log = logging.getLogger(__name__)
@@ -16,11 +16,33 @@ log = logging.getLogger(__name__)
NOT_SET = type(str('NOT_SET'), (object,), {})
T = typing.TypeVar('T')
KB = 2 ** 10
MB = 2 ** 20
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
API_ADDRESS = 'lbryapi'
APP_NAME = 'LBRY'
BLOBFILES_DIR = 'blobfiles'
CRYPTSD_FILE_EXTENSION = '.cryptsd'
CURRENCIES = {
'BTC': {'type': 'crypto'},
'LBC': {'type': 'crypto'},
'USD': {'type': 'fiat'},
}
ICON_PATH = 'icons' if 'win' in sys.platform else 'app.icns'
LOG_FILE_NAME = 'lbrynet.log'
LOG_POST_URL = 'https://lbry.io/log-upload'
MAX_BLOB_REQUEST_SIZE = 64 * KB
MAX_HANDSHAKE_SIZE = 64 * KB
MAX_REQUEST_SIZE = 64 * KB
MAX_RESPONSE_INFO_SIZE = 64 * KB
MAX_BLOB_INFOS_TO_REQUEST = 20
PROTOCOL_PREFIX = 'lbry'
SLACK_WEBHOOK = (
'nUE0pUZ6Yl9bo29epl5moTSwnl5wo20ip2IlqzywMKZiIQSFZR5'
'AHx4mY0VmF0WQZ1ESEP9kMHZlp1WzJwWOoKN3ImR1M2yUAaMyqGZ='
)
HEADERS_FILE_SHA256_CHECKSUM = (
366295, 'b0c8197153a33ccbc52fb81a279588b6015b68b7726f73f6a2b81f7e25bfe4b9'
)
@@ -29,24 +51,14 @@ HEADERS_FILE_SHA256_CHECKSUM = (
class Setting(typing.Generic[T]):
def __init__(self, doc: str, default: typing.Optional[T] = None,
previous_names: typing.Optional[typing.List[str]] = None,
metavar: typing.Optional[str] = None):
previous_names: typing.Optional[typing.List[str]] = None):
self.doc = doc
self.default = default
self.previous_names = previous_names or []
self.metavar = metavar
def __set_name__(self, owner, name):
self.name = name
@property
def cli_name(self):
return f"--{self.name.replace('_', '-')}"
@property
def no_cli_name(self):
return f"--no-{self.name.replace('_', '-')}"
def __get__(self, obj: typing.Optional['BaseConfig'], owner) -> T:
if obj is None:
return self
@@ -74,14 +86,6 @@ class Setting(typing.Generic[T]):
def serialize(self, value):
return value
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
help=self.doc,
metavar=self.metavar,
default=NOT_SET
)
class String(Setting[str]):
def validate(self, val):
@@ -94,39 +98,18 @@ class Integer(Setting[int]):
assert isinstance(val, int), \
f"Setting '{self.name}' must be an integer."
def deserialize(self, value):
return int(value)
class Float(Setting[float]):
def validate(self, val):
assert isinstance(val, float), \
f"Setting '{self.name}' must be a decimal."
def deserialize(self, value):
return float(value)
class Toggle(Setting[bool]):
def validate(self, val):
assert isinstance(val, bool), \
f"Setting '{self.name}' must be a true/false value."
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
help=self.doc,
action="store_true",
default=NOT_SET
)
parser.add_argument(
self.no_cli_name,
help=f"Opposite of --{self.cli_name}",
dest=self.name,
action="store_false",
default=NOT_SET
)
class Path(String):
def __init__(self, doc: str, default: str = '', *args, **kwargs):
@@ -142,54 +125,16 @@ class Path(String):
class MaxKeyFee(Setting[dict]):
def validate(self, value):
if value is not None:
assert isinstance(value, dict) and set(value) == {'currency', 'amount'}, \
f"Setting '{self.name}' must be a dict like \"{{'amount': 50.0, 'currency': 'USD'}}\"."
if value["currency"] not in CURRENCIES:
raise InvalidCurrencyError(value["currency"])
@staticmethod
def _parse_list(l):
assert len(l) == 2, 'Max key fee is made up of two values: "AMOUNT CURRENCY".'
try:
amount = float(l[0])
except ValueError:
raise AssertionError('First value in max key fee is a decimal: "AMOUNT CURRENCY"')
currency = str(l[1]).upper()
assert isinstance(value, dict), \
f"Setting '{self.name}' must be of the format \"{'currency': 'USD', 'amount': 50.0}\"."
assert set(value) == {'currency', 'amount'}, \
f"Setting '{self.name}' must contain a 'currency' and an 'amount' field."
currency = str(value["currency"]).upper()
if currency not in CURRENCIES:
raise InvalidCurrencyError(currency)
return {'amount': amount, 'currency': currency}
def deserialize(self, value):
if value is None:
return
if isinstance(value, dict):
return {
'currency': value['currency'],
'amount': float(value['amount']),
}
if isinstance(value, str):
value = value.split()
if isinstance(value, list):
return self._parse_list(value)
raise AssertionError('Invalid max key fee.')
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
help=self.doc,
nargs=2,
metavar=('AMOUNT', 'CURRENCY'),
default=NOT_SET
)
parser.add_argument(
self.no_cli_name,
help=f"Disable maximum key fee check.",
dest=self.name,
const=None,
action="store_const",
default=NOT_SET
)
serialize = staticmethod(json.dumps)
deserialize = staticmethod(json.loads)
class Servers(Setting[list]):
@@ -225,14 +170,6 @@ class Servers(Setting[list]):
return [f"{host}:{port}" for host, port in value]
return value
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
nargs="*",
help=self.doc,
default=NOT_SET
)
class Strings(Setting[list]):
@@ -260,23 +197,14 @@ class EnvironmentAccess:
class ArgumentAccess:
def __init__(self, config: 'BaseConfig', args: dict):
self.configuration = config
self.args = {}
if args:
self.load(args)
def load(self, args):
for setting in self.configuration.get_settings():
value = getattr(args, setting.name, NOT_SET)
if value != NOT_SET:
self.args[setting.name] = setting.deserialize(value)
def __init__(self, args: dict):
self.args = args
def __contains__(self, item: str):
return item in self.args
return getattr(self.args, item, None) is not None
def __getitem__(self, item: str):
return self.args[item]
return getattr(self.args, item)
class ConfigFileAccess:
@@ -342,7 +270,7 @@ class ConfigFileAccess:
class BaseConfig:
config = Path("Path to configuration file.", metavar='FILE')
config = Path("Path to configuration file.")
def __init__(self, **kwargs):
self.runtime = {} # set internally or by various API calls
@@ -380,8 +308,7 @@ class BaseConfig:
@classmethod
def get_settings(cls):
for attr in dir(cls):
setting = getattr(cls, attr)
for setting in cls.__dict__.values():
if isinstance(setting, Setting):
yield setting
@@ -404,12 +331,22 @@ class BaseConfig:
return conf
@classmethod
def contribute_to_argparse(cls, parser: ArgumentParser):
def contribute_args(cls, parser: ArgumentParser):
for setting in cls.get_settings():
setting.contribute_to_argparse(parser)
if isinstance(setting, Toggle):
parser.add_argument(
f"--{setting.name.replace('_', '-')}",
help=setting.doc,
action="store_true"
)
else:
parser.add_argument(
f"--{setting.name.replace('_', '-')}",
help=setting.doc
)
def set_arguments(self, args):
self.arguments = ArgumentAccess(self, args)
self.arguments = ArgumentAccess(args)
def set_environment(self, environ=None):
self.environment = EnvironmentAccess(environ or os.environ)
@@ -433,7 +370,7 @@ class BaseConfig:
class CLIConfig(BaseConfig):
api = String('Host name and port for lbrynet daemon API.', 'localhost:5279', metavar='HOST:PORT')
api = String('Host name and port for lbrynet daemon API.', 'localhost:5279')
@property
def api_connection_url(self) -> str:
@@ -449,121 +386,71 @@ class CLIConfig(BaseConfig):
class Config(CLIConfig):
# directories
data_dir = Path("Directory path to store blobs.", metavar='DIR')
download_dir = Path(
"Directory path to place assembled files downloaded from LBRY.",
previous_names=['download_directory'], metavar='DIR'
)
data_dir = Path("Directory path to store blobs.")
download_dir = Path("Directory path to place assembled files downloaded from LBRY.")
wallet_dir = Path(
"Directory containing a 'wallets' subdirectory with 'default_wallet' file.",
previous_names=['lbryum_wallet_dir'], metavar='DIR'
previous_names=['lbryum_wallet_dir']
)
# network
use_upnp = Toggle(
"Use UPnP to setup temporary port redirects for the DHT and the hosting of blobs. If you manually forward"
"ports or have firewall rules you likely want to disable this.", True
)
udp_port = Integer("UDP port for communicating on the LBRY DHT", 4444, previous_names=['dht_node_port'])
tcp_port = Integer("TCP port to listen for incoming blob requests", 3333, previous_names=['peer_port'])
network_interface = String("Interface to use for the DHT and blob exchange", '0.0.0.0')
# routing table
split_buckets_under_index = Integer(
"Routing table bucket index below which we always split the bucket if given a new key to add to it and "
"the bucket is full. As this value is raised the depth of the routing table (and number of peers in it) "
"will increase. This setting is used by seed nodes, you probably don't want to change it during normal "
"use.", 1
)
# protocol timeouts
download_timeout = Float("Cumulative timeout for a stream to begin downloading before giving up", 30.0)
blob_download_timeout = Float("Timeout to download a blob from a peer", 30.0)
peer_connect_timeout = Float("Timeout to establish a TCP connection to a peer", 3.0)
node_rpc_timeout = Float("Timeout when making a DHT request", constants.rpc_timeout)
# blob announcement and download
save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True)
blob_lru_cache_size = Integer(
"LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when "
"replying to a range request. Set to 0 to disable.", 32
)
announce_head_and_sd_only = Toggle(
"Announce only the descriptor and first (rather than all) data blob for a stream to the DHT", True,
previous_names=['announce_head_blobs_only']
)
concurrent_blob_announcers = Integer(
"Number of blobs to iteratively announce at once, set to 0 to disable", 10,
previous_names=['concurrent_announcers']
)
max_connections_per_download = Integer(
"Maximum number of peers to connect to while downloading a blob", 4,
previous_names=['max_connections_per_stream']
)
fixed_peer_delay = Float(
"Amount of seconds before adding the reflector servers as potential peers to download from in case dht"
"peers are not found or are slow", 2.0
)
max_key_fee = MaxKeyFee(
"Don't download streams with fees exceeding this amount", {'currency': 'USD', 'amount': 50.0}
)
# reflector settings
reflect_streams = Toggle(
"Upload completed streams (published and downloaded) reflector in order to re-host them", True,
previous_names=['reflect_uploads']
)
concurrent_reflector_uploads = Integer(
"Maximum number of streams to upload to a reflector server at a time", 10
)
# servers
reflector_servers = Servers("Reflector re-hosting servers", [
('reflector.lbry.io', 5566)
])
lbryum_servers = Servers("SPV wallet servers", [
('lbryumx1.lbry.com', 50001), # US EAST
('lbryumx2.lbry.com', 50001), # US EAST
('lbryumx4.lbry.com', 50001) #EU
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.com', 4444), # US EAST
('lbrynet2.lbry.com', 4444), # US WEST
('lbrynet3.lbry.com', 4444), # EU
('lbrynet4.lbry.com', 4444) # ASIA
])
# comment server uses a single string because requests are made to the /api resource
comment_server = String("Server to store user-generated metadata.", "http://comments.lbry.com:2903/api")
# blockchain
blockchain_name = String("Blockchain name - lbrycrd_main, lbrycrd_regtest, or lbrycrd_testnet", 'lbrycrd_main')
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
cache_time = Integer("Time to cache resolved claims", 150) # TODO: use this
# daemon
save_files = Toggle("Save downloaded files when calling `get` by default", True)
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", True,
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
# media server
streaming_server = String('Host name and port to serve streaming media over range requests',
'localhost:5280', metavar='HOST:PORT')
streaming_get = Toggle("Enable the /get endpoint for the streaming media server. "
"Disable to prevent new streams from being added.", True)
@property
def streaming_host(self):
return self.streaming_server.split(':')[0]
@property
def streaming_port(self):
return int(self.streaming_server.split(':')[1])
# claims set to expire within this many blocks will be
# automatically renewed after startup (if set to 0, renews
# will not be made automatically)
auto_renew_claim_height_delta = Integer("", 0)
cache_time = Integer("", 150)
data_rate = Float("points/megabyte", .0001)
delete_blobs_on_remove = Toggle("", True)
dht_node_port = Integer("", 4444)
download_timeout = Integer("", 180)
download_mirrors = Servers("", [
('blobs.lbry.io', 80)
])
is_generous_host = Toggle("", True)
announce_head_blobs_only = Toggle("", True)
concurrent_announcers = Integer("", 10)
known_dht_nodes = Servers("", [
('lbrynet1.lbry.io', 4444), # US EAST
('lbrynet2.lbry.io', 4444), # US WEST
('lbrynet3.lbry.io', 4444), # EU
('lbrynet4.lbry.io', 4444) # ASIA
])
max_connections_per_stream = Integer("", 5)
seek_head_blob_first = Toggle("", True)
# TODO: writing json on the cmd line is a pain, come up with a nicer
# parser for this data structure. maybe 'USD:25'
max_key_fee = MaxKeyFee("", {'currency': 'USD', 'amount': 50.0})
disable_max_key_fee = Toggle("", False)
min_info_rate = Float("points/1000 infos", .02)
min_valuable_hash_rate = Float("points/1000 infos", .05)
min_valuable_info_rate = Float("points/1000 infos", .05)
peer_port = Integer("", 3333)
pointtrader_server = String("", 'http://127.0.0.1:2424')
reflector_port = Integer("", 5566)
# if reflect_uploads is True, send files to reflector after publishing (as well as a periodic check in the
# event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0)
reflect_uploads = Toggle("", True)
auto_re_reflect_interval = Integer("set to 0 to disable", 86400)
reflector_servers = Servers("", [
('reflector.lbry.io', 5566)
])
run_reflector_server = Toggle("adds reflector to components_to_skip unless True", False)
sd_download_timeout = Integer("", 3)
peer_search_timeout = Integer("", 60)
use_upnp = Toggle("", True)
use_keyring = Toggle("", False)
blockchain_name = String("", 'lbrycrd_main')
lbryum_servers = Servers("", [
('lbryumx1.lbry.io', 50001),
('lbryumx2.lbry.io', 50001)
])
s3_headers_depth = Integer("download headers from s3 when the local height is more than 10 chunks behind", 96 * 10)
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
def __init__(self, **kwargs):
super().__init__(**kwargs)
@@ -619,10 +506,11 @@ def get_linux_directories() -> typing.Tuple[str, str, str]:
try:
with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg:
down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1)
down_dir = re.sub('\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir)
download_dir = re.sub('\"', '', down_dir)
down_dir = re.sub('\$HOME', os.getenv('HOME'), down_dir)
download_dir = re.sub('\"', '', down_dir)
except EnvironmentError:
download_dir = os.getenv('XDG_DOWNLOAD_DIR')
if not download_dir:
download_dir = os.path.expanduser('~/Downloads')
-89
View File
@@ -1,89 +0,0 @@
import asyncio
import typing
import collections
import logging
log = logging.getLogger(__name__)
CONNECTED_EVENT = "connected"
DISCONNECTED_EVENT = "disconnected"
TRANSFERRED_EVENT = "transferred"
class ConnectionManager:
def __init__(self, loop: asyncio.AbstractEventLoop):
self.loop = loop
self.incoming_connected: typing.Set[str] = set()
self.incoming: typing.DefaultDict[str, int] = collections.defaultdict(int)
self.outgoing_connected: typing.Set[str] = set()
self.outgoing: typing.DefaultDict[str, int] = collections.defaultdict(int)
self._status = {}
self._task: typing.Optional[asyncio.Task] = None
@property
def status(self):
return self._status
def sent_data(self, host_and_port: str, size: int):
self.outgoing[host_and_port] += size
def received_data(self, host_and_port: str, size: int):
self.incoming[host_and_port] += size
def connection_made(self, host_and_port: str):
self.outgoing_connected.add(host_and_port)
def connection_received(self, host_and_port: str):
# self.incoming_connected.add(host_and_port)
pass
def outgoing_connection_lost(self, host_and_port: str):
if host_and_port in self.outgoing_connected:
self.outgoing_connected.remove(host_and_port)
def incoming_connection_lost(self, host_and_port: str):
if host_and_port in self.incoming_connected:
self.incoming_connected.remove(host_and_port)
async def _update(self):
self._status = {
'incoming_bps': {},
'outgoing_bps': {},
'total_incoming_mbps': 0.0,
'total_outgoing_mbps': 0.0,
}
while True:
last = self.loop.time()
await asyncio.sleep(1, loop=self.loop)
self._status['incoming_bps'].clear()
self._status['outgoing_bps'].clear()
while self.outgoing:
k, v = self.outgoing.popitem()
self._status['outgoing_bps'][k] = v
while self.incoming:
k, v = self.incoming.popitem()
self._status['incoming_bps'][k] = v
now = self.loop.time()
self._status['total_outgoing_mbps'] = int(sum(list(self._status['outgoing_bps'].values()))
/ (now - last)) / 1000000.0
self._status['total_incoming_mbps'] = int(sum(list(self._status['incoming_bps'].values()))
/ (now - last)) / 1000000.0
self._status['time'] = now
def stop(self):
if self._task:
self._task.cancel()
self._task = None
self.outgoing.clear()
self.outgoing_connected.clear()
self.incoming.clear()
self.incoming_connected.clear()
self._status.clear()
def start(self):
self.stop()
self._task = self.loop.create_task(self._update())
-4
View File
@@ -1,8 +1,4 @@
import hashlib
from cryptography.hazmat.backends import default_backend
backend = default_backend()
def get_lbry_hash_obj():
+7
View File
@@ -0,0 +1,7 @@
Francois Aucamp <faucamp@csir.co.za>
Thanks goes to the following people for providing patches/suggestions/tests:
Neil Kleynhans <ntkleynhans@csir.co.za>
Haiyang Ma <haiyang.ma@maidsafe.net>
Bryan McAlister <bmcalister@csir.co.za>
+165
View File
@@ -0,0 +1,165 @@
GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.
-59
View File
@@ -1,59 +0,0 @@
import asyncio
import typing
import logging
if typing.TYPE_CHECKING:
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.storage import SQLiteStorage
log = logging.getLogger(__name__)
class BlobAnnouncer:
def __init__(self, loop: asyncio.BaseEventLoop, node: 'Node', storage: 'SQLiteStorage'):
self.loop = loop
self.node = node
self.storage = storage
self.announce_task: asyncio.Task = None
self.announce_queue: typing.List[str] = []
async def _submit_announcement(self, blob_hash):
try:
peers = len(await self.node.announce_blob(blob_hash))
if peers > 4:
return blob_hash
else:
log.warning("failed to announce %s, could only find %d peers, retrying soon.", blob_hash[:8], peers)
except Exception as err:
if isinstance(err, asyncio.CancelledError):
raise err
log.warning("error announcing %s: %s", blob_hash[:8], str(err))
async def _announce(self, batch_size: typing.Optional[int] = 10):
while batch_size:
if not self.node.joined.is_set():
await self.node.joined.wait()
await asyncio.sleep(60, loop=self.loop)
if not self.node.protocol.routing_table.get_peers():
log.warning("No peers in DHT, announce round skipped")
continue
self.announce_queue.extend(await self.storage.get_blobs_to_announce())
log.debug("announcer task wake up, %d blobs to announce", len(self.announce_queue))
while len(self.announce_queue):
log.info("%i blobs to announce", len(self.announce_queue))
announced = await asyncio.gather(*[
self._submit_announcement(
self.announce_queue.pop()) for _ in range(batch_size) if self.announce_queue
], loop=self.loop)
announced = list(filter(None, announced))
if announced:
await self.storage.update_last_announced_blobs(announced)
log.info("announced %i blobs", len(announced))
def start(self, batch_size: typing.Optional[int] = 10):
assert not self.announce_task or self.announce_task.done(), "already running"
self.announce_task = self.loop.create_task(self._announce(batch_size))
def stop(self):
if self.announce_task and not self.announce_task.done():
self.announce_task.cancel()
+81
View File
@@ -0,0 +1,81 @@
import logging
log = logging.getLogger()
MIN_DELAY = 0.0
MAX_DELAY = 0.01
DELAY_INCREMENT = 0.0001
QUEUE_SIZE_THRESHOLD = 100
class CallLaterManager:
def __init__(self, callLater):
"""
:param callLater: (IReactorTime.callLater)
"""
self._callLater = callLater
self._pendingCallLaters = []
self._delay = MIN_DELAY
def get_min_delay(self):
self._pendingCallLaters = [cl for cl in self._pendingCallLaters if cl.active()]
queue_size = len(self._pendingCallLaters)
if queue_size > QUEUE_SIZE_THRESHOLD:
self._delay = min((self._delay + DELAY_INCREMENT), MAX_DELAY)
else:
self._delay = max((self._delay - 2.0 * DELAY_INCREMENT), MIN_DELAY)
return self._delay
def _cancel(self, call_later):
"""
:param call_later: DelayedCall
:return: (callable) canceller function
"""
def cancel(reason=None):
"""
:param reason: reason for cancellation, this is returned after cancelling the DelayedCall
:return: reason
"""
if call_later.active():
call_later.cancel()
if call_later in self._pendingCallLaters:
self._pendingCallLaters.remove(call_later)
return reason
return cancel
def stop(self):
"""
Cancel any callLaters that are still running
"""
from twisted.internet import defer
while self._pendingCallLaters:
canceller = self._cancel(self._pendingCallLaters[0])
try:
canceller()
except (defer.CancelledError, defer.AlreadyCalledError, ValueError):
pass
def call_later(self, when, what, *args, **kwargs):
"""
Schedule a call later and get a canceller callback function
:param when: (float) delay in seconds
:param what: (callable)
:param args: (*tuple) args to be passed to the callable
:param kwargs: (**dict) kwargs to be passed to the callable
:return: (tuple) twisted.internet.base.DelayedCall object, canceller function
"""
call_later = self._callLater(when, what, *args, **kwargs)
canceller = self._cancel(call_later)
self._pendingCallLaters.append(call_later)
return call_later, canceller
def call_soon(self, what, *args, **kwargs):
delay = self.get_min_delay()
return self.call_later(delay, what, *args, **kwargs)
+57 -38
View File
@@ -1,42 +1,61 @@
import hashlib
import os
#!/usr/bin/env python
#
# This library is free software, distributed under the terms of
# the GNU Lesser General Public License Version 3, or any later version.
# See the COPYING file included in this archive
#
# The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net
hash_class = hashlib.sha384
hash_length = hash_class().digest_size
hash_bits = hash_length * 8
alpha = 5
""" This module defines the charaterizing constants of the Kademlia network
C{checkRefreshInterval} and C{udpDatagramMaxSize} are implementation-specific
constants, and do not affect general Kademlia operation.
"""
######### KADEMLIA CONSTANTS ###########
#: Small number Representing the degree of parallelism in network calls
alpha = 3
#: Maximum number of contacts stored in a bucket; this should be an even number
k = 8
split_buckets_under_index = 1
replacement_cache_size = 8
rpc_timeout = 5.0
rpc_attempts = 5
rpc_attempts_pruning_window = 600
iterative_lookup_delay = rpc_timeout / 2.0 # TODO: use config val / 2 if rpc timeout is provided
refresh_interval = 3600 # 1 hour
replicate_interval = refresh_interval
data_expiration = 86400 # 24 hours
token_secret_refresh_interval = 300 # 5 minutes
maybe_ping_delay = 300 # 5 minutes
check_refresh_interval = refresh_interval / 5
max_datagram_size = 8192 # 8 KB
#: Maximum number of contacts stored in the replacement cache
replacementCacheSize = 8
#: Timeout for network operations (in seconds)
rpcTimeout = 5
# number of rpc attempts to make before a timeout results in the node being removed as a contact
rpcAttempts = 5
# time window to count failures (in seconds)
rpcAttemptsPruningTimeWindow = 600
# Delay between iterations of iterative node lookups (for loose parallelism) (in seconds)
iterativeLookupDelay = rpcTimeout / 2
#: If a k-bucket has not been used for this amount of time, refresh it (in seconds)
refreshTimeout = 3600 # 1 hour
#: The interval at which nodes replicate (republish/refresh) data they are holding
replicateInterval = refreshTimeout
# The time it takes for data to expire in the network; the original publisher of the data
# will also republish the data at this time if it is still valid
dataExpireTimeout = 86400 # 24 hours
tokenSecretChangeInterval = 300 # 5 minutes
######## IMPLEMENTATION-SPECIFIC CONSTANTS ###########
#: The interval for the node to check whether any buckets need refreshing
checkRefreshInterval = refreshTimeout / 5
#: Max size of a single UDP datagram, in bytes. If a message is larger than this, it will
#: be spread across several UDP packets.
udpDatagramMaxSize = 8192 # 8 KB
key_bits = 384
rpc_id_length = 20
protocol_version = 1
bottom_out_limit = 3
msg_size_limit = max_datagram_size - 26
def digest(data: bytes) -> bytes:
h = hash_class()
h.update(data)
return h.digest()
def generate_id(num=None) -> bytes:
if num is not None:
return digest(str(num).encode())
else:
return digest(os.urandom(32))
def generate_rpc_id(num=None) -> bytes:
return generate_id(num)[:rpc_id_length]
protocolVersion = 1
+189
View File
@@ -0,0 +1,189 @@
import ipaddress
from binascii import hexlify
from functools import reduce
from lbrynet.dht import constants
def is_valid_ipv4(address):
try:
ip = ipaddress.ip_address(address)
return ip.version == 4
except ipaddress.AddressValueError:
return False
class _Contact:
""" Encapsulation for remote contact
This class contains information on a single remote contact, and also
provides a direct RPC API to the remote node which it represents
"""
def __init__(self, contactManager, id, ipAddress, udpPort, networkProtocol, firstComm):
if id is not None:
if not len(id) == constants.key_bits // 8:
raise ValueError("invalid node id: {}".format(hexlify(id).decode()))
if not 0 <= udpPort <= 65536:
raise ValueError("invalid port")
if not is_valid_ipv4(ipAddress):
raise ValueError("invalid ip address")
self._contactManager = contactManager
self._id = id
self.address = ipAddress
self.port = udpPort
self._networkProtocol = networkProtocol
self.commTime = firstComm
self.getTime = self._contactManager._get_time
self.lastReplied = None
self.lastRequested = None
self.protocolVersion = 0
self._token = (None, 0) # token, timestamp
def update_token(self, token):
self._token = token, self.getTime() if token else 0
@property
def token(self):
# expire the token 1 minute early to be safe
return self._token[0] if self._token[1] + 240 > self.getTime() else None
@property
def lastInteracted(self):
return max(self.lastRequested or 0, self.lastReplied or 0, self.lastFailed or 0)
@property
def id(self):
return self._id
def log_id(self, short=True):
if not self.id:
return "not initialized"
id_hex = hexlify(self.id)
return id_hex if not short else id_hex[:8]
@property
def failedRPCs(self):
return len(self.failures)
@property
def lastFailed(self):
return (self.failures or [None])[-1]
@property
def failures(self):
return self._contactManager._rpc_failures.get((self.address, self.port), [])
@property
def contact_is_good(self):
"""
:return: False if contact is bad, None if contact is unknown, or True if contact is good
"""
failures = self.failures
now = self.getTime()
delay = constants.checkRefreshInterval
if failures:
if self.lastReplied and len(failures) >= 2 and self.lastReplied < failures[-2]:
return False
elif self.lastReplied and len(failures) >= 2 and self.lastReplied > failures[-2]:
pass # handled below
elif len(failures) >= 2:
return False
if self.lastReplied and self.lastReplied > now - delay:
return True
if self.lastReplied and self.lastRequested and self.lastRequested > now - delay:
return True
return None
def __eq__(self, other):
if not isinstance(other, _Contact):
raise TypeError("invalid type to compare with Contact: %s" % str(type(other)))
return (self.id, self.address, self.port) == (other.id, other.address, other.port)
def __hash__(self):
return hash((self.id, self.address, self.port))
def compact_ip(self):
compact_ip = reduce(
lambda buff, x: buff + bytearray([int(x)]), self.address.split('.'), bytearray())
return compact_ip
def set_id(self, id):
if not self._id:
self._id = id
def update_last_replied(self):
self.lastReplied = int(self.getTime())
def update_last_requested(self):
self.lastRequested = int(self.getTime())
def update_last_failed(self):
failures = self._contactManager._rpc_failures.get((self.address, self.port), [])
failures.append(self.getTime())
self._contactManager._rpc_failures[(self.address, self.port)] = failures
def update_protocol_version(self, version):
self.protocolVersion = version
def __str__(self):
return '<%s.%s object; IP address: %s, UDP port: %d>' % (
self.__module__, self.__class__.__name__, self.address, self.port)
def __getattr__(self, name):
""" This override allows the host node to call a method of the remote
node (i.e. this contact) as if it was a local function.
For instance, if C{remoteNode} is a instance of C{Contact}, the
following will result in C{remoteNode}'s C{test()} method to be
called with argument C{123}::
remoteNode.test(123)
Such a RPC method call will return a Deferred, which will callback
when the contact responds with the result (or an error occurs).
This happens via this contact's C{_networkProtocol} object (i.e. the
host Node's C{_protocol} object).
"""
if name not in ['ping', 'findValue', 'findNode', 'store']:
raise AttributeError("unknown command: %s" % name)
def _sendRPC(*args, **kwargs):
return self._networkProtocol.sendRPC(self, name.encode(), args)
return _sendRPC
class ContactManager:
def __init__(self, get_time=None):
if not get_time:
from twisted.internet import reactor
get_time = reactor.seconds
self._get_time = get_time
self._contacts = {}
self._rpc_failures = {}
def get_contact(self, id, address, port):
for contact in self._contacts.values():
if contact.id == id and contact.address == address and contact.port == port:
return contact
def make_contact(self, id, ipAddress, udpPort, networkProtocol, firstComm=0):
contact = self.get_contact(id, ipAddress, udpPort)
if contact:
return contact
contact = _Contact(self, id, ipAddress, udpPort, networkProtocol, firstComm or self._get_time())
self._contacts[(id, ipAddress, udpPort)] = contact
return contact
def is_ignored(self, origin_tuple):
failed_rpc_count = len(self._prune_failures(origin_tuple))
return failed_rpc_count > constants.rpcAttempts
def _prune_failures(self, origin_tuple):
# Prunes recorded failures to the last time window of attempts
pruning_limit = self._get_time() - constants.rpcAttemptsPruningTimeWindow
pruned = list(filter(lambda t: t >= pruning_limit, self._rpc_failures.get(origin_tuple, [])))
self._rpc_failures[origin_tuple] = pruned
return pruned
+66
View File
@@ -0,0 +1,66 @@
from collections import UserDict
from lbrynet.dht import constants
class DictDataStore(UserDict):
""" A datastore using an in-memory Python dictionary """
#implements(IDataStore)
def __init__(self, getTime=None):
# Dictionary format:
# { <key>: (<contact>, <value>, <lastPublished>, <originallyPublished> <originalPublisherID>) }
super().__init__()
if not getTime:
from twisted.internet import reactor
getTime = reactor.seconds
self._getTime = getTime
self.completed_blobs = set()
def filter_bad_and_expired_peers(self, key):
"""
Returns only non-expired and unknown/good peers
"""
return filter(
lambda peer:
self._getTime() - peer[3] < constants.dataExpireTimeout and peer[0].contact_is_good is not False,
self[key]
)
def filter_expired_peers(self, key):
"""
Returns only non-expired peers
"""
return filter(lambda peer: self._getTime() - peer[3] < constants.dataExpireTimeout, self[key])
def removeExpiredPeers(self):
expired_keys = []
for key in self.keys():
unexpired_peers = list(self.filter_expired_peers(key))
if not unexpired_peers:
expired_keys.append(key)
else:
self[key] = unexpired_peers
for key in expired_keys:
del self[key]
def hasPeersForBlob(self, key):
return bool(key in self and len(tuple(self.filter_bad_and_expired_peers(key))))
def addPeerToBlob(self, contact, key, compact_address, lastPublished, originallyPublished, originalPublisherID):
if key in self:
if compact_address not in map(lambda store_tuple: store_tuple[1], self[key]):
self[key].append(
(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)
)
else:
self[key] = [(contact, compact_address, lastPublished, originallyPublished, originalPublisherID)]
def getPeersForBlob(self, key):
return [] if key not in self else [val[1] for val in self.filter_bad_and_expired_peers(key)]
def getStoringContacts(self):
contacts = set()
for key in self:
for values in self[key]:
contacts.add(values[0])
return list(contacts)
@@ -8,18 +8,20 @@ class Distance:
we pre-calculate the value of that point.
"""
def __init__(self, key: bytes):
if len(key) != constants.hash_length:
raise ValueError(f"invalid key length: {len(key)}")
def __init__(self, key):
if len(key) != constants.key_bits // 8:
raise ValueError("invalid key length: %i" % len(key))
self.key = key
self.val_key_one = int.from_bytes(key, 'big')
def __call__(self, key_two: bytes) -> int:
if len(key_two) != constants.hash_length:
raise ValueError(f"invalid length of key to compare: {len(key_two)}")
def __call__(self, key_two):
val_key_two = int.from_bytes(key_two, 'big')
return self.val_key_one ^ val_key_two
def is_closer(self, a: bytes, b: bytes) -> bool:
def is_closer(self, a, b):
"""Returns true is `a` is closer to `key` than `b` is"""
return self(a) < self(b)
def to_contact(self, contact):
"""A convenience function for calculating the distance to a contact"""
return self(contact.id)
@@ -1,8 +1,8 @@
import typing
from lbrynet.dht.error import DecodeError
def _bencode(data: typing.Union[int, bytes, bytearray, str, list, tuple, dict]) -> bytes:
def bencode(data):
""" Encoder implementation of the Bencode algorithm (Bittorrent). """
if isinstance(data, int):
return b'i%de' % data
elif isinstance(data, (bytes, bytearray)):
@@ -12,20 +12,31 @@ def _bencode(data: typing.Union[int, bytes, bytearray, str, list, tuple, dict])
elif isinstance(data, (list, tuple)):
encoded_list_items = b''
for item in data:
encoded_list_items += _bencode(item)
encoded_list_items += bencode(item)
return b'l%se' % encoded_list_items
elif isinstance(data, dict):
encoded_dict_items = b''
keys = data.keys()
for key in sorted(keys):
encoded_dict_items += _bencode(key)
encoded_dict_items += _bencode(data[key])
encoded_dict_items += bencode(key)
encoded_dict_items += bencode(data[key])
return b'd%se' % encoded_dict_items
else:
raise TypeError(f"Cannot bencode {type(data)}")
raise TypeError("Cannot bencode '%s' object" % type(data))
def _bdecode(data: bytes, start_index: int = 0) -> typing.Tuple[typing.Union[int, bytes, list, tuple, dict], int]:
def bdecode(data):
""" Decoder implementation of the Bencode algorithm. """
assert type(data) == bytes # fixme: _maybe_ remove this after porting
if len(data) == 0:
raise DecodeError('Cannot decode empty string')
try:
return _decode_recursive(data)[0]
except ValueError as e:
raise DecodeError(str(e))
def _decode_recursive(data, start_index=0):
if data[start_index] == ord('i'):
end_pos = data[start_index:].find(b'e') + start_index
return int(data[start_index + 1:end_pos]), end_pos + 1
@@ -33,44 +44,32 @@ def _bdecode(data: bytes, start_index: int = 0) -> typing.Tuple[typing.Union[int
start_index += 1
decoded_list = []
while data[start_index] != ord('e'):
list_data, start_index = _bdecode(data, start_index)
list_data, start_index = _decode_recursive(data, start_index)
decoded_list.append(list_data)
return decoded_list, start_index + 1
elif data[start_index] == ord('d'):
start_index += 1
decoded_dict = {}
while data[start_index] != ord('e'):
key, start_index = _bdecode(data, start_index)
value, start_index = _bdecode(data, start_index)
key, start_index = _decode_recursive(data, start_index)
value, start_index = _decode_recursive(data, start_index)
decoded_dict[key] = value
return decoded_dict, start_index
elif data[start_index] == ord('f'):
# This (float data type) is a non-standard extension to the original Bencode algorithm
end_pos = data[start_index:].find(b'e') + start_index
return float(data[start_index + 1:end_pos]), end_pos + 1
elif data[start_index] == ord('n'):
# This (None/NULL data type) is a non-standard extension
# to the original Bencode algorithm
return None, start_index + 1
else:
split_pos = data[start_index:].find(b':') + start_index
try:
length = int(data[start_index:split_pos])
except (ValueError, TypeError) as err:
raise DecodeError(err)
except ValueError:
raise DecodeError()
start_index = split_pos + 1
end_pos = start_index + length
b = data[start_index:end_pos]
return b, end_pos
def bencode(data: typing.Dict) -> bytes:
if not isinstance(data, dict):
raise TypeError()
return _bencode(data)
def bdecode(data: bytes, allow_non_dict_return: typing.Optional[bool] = False) -> typing.Dict:
assert type(data) == bytes, DecodeError(f"invalid data type: {str(type(data))}")
if len(data) == 0:
raise DecodeError('Cannot decode empty string')
try:
result = _bdecode(data)[0]
if not allow_non_dict_return and not isinstance(result, dict):
raise ValueError(f'expected dict, got {type(result)}')
return result
except (ValueError, TypeError) as err:
raise DecodeError(err)
+26 -6
View File
@@ -1,23 +1,43 @@
class BaseKademliaException(Exception):
pass
import binascii
#import exceptions
# this is a dict of {"exceptions.<exception class name>": exception class} items used to raise
# remote built-in exceptions locally
BUILTIN_EXCEPTIONS = {
# "exceptions.%s" % e: getattr(exceptions, e) for e in dir(exceptions) if not e.startswith("_")
}
class DecodeError(BaseKademliaException):
class DecodeError(Exception):
"""
Should be raised by an C{Encoding} implementation if decode operation
fails
"""
class BucketFull(BaseKademliaException):
class BucketFull(Exception):
"""
Raised when the bucket is full
"""
class RemoteException(BaseKademliaException):
class UnknownRemoteException(Exception):
pass
class TransportNotConnected(BaseKademliaException):
class TimeoutError(Exception):
""" Raised when a RPC times out """
def __init__(self, remote_contact_id):
# remote_contact_id is a binary blob so we need to convert it
# into something more readable
if remote_contact_id:
msg = 'Timeout connecting to {}'.format(binascii.hexlify(remote_contact_id))
else:
msg = 'Timeout connecting to uninitialized node'
super().__init__(msg)
self.remote_contact_id = remote_contact_id
class TransportNotConnected(Exception):
pass
+116
View File
@@ -0,0 +1,116 @@
from zope.interface import Interface
class IDataStore(Interface):
""" Interface for classes implementing physical storage (for data
published via the "STORE" RPC) for the Kademlia DHT
@note: This provides an interface for a dict-like object
"""
def keys(self):
""" Return a list of the keys in this data store """
def removeExpiredPeers(self):
pass
def hasPeersForBlob(self, key):
pass
def addPeerToBlob(self, key, value, lastPublished, originallyPublished, originalPublisherID):
pass
def getPeersForBlob(self, key):
pass
def removePeer(self, key):
pass
class IRoutingTable(Interface):
""" Interface for RPC message translators/formatters
Classes inheriting from this should provide a suitable routing table for
a parent Node object (i.e. the local entity in the Kademlia network)
"""
def __init__(self, parentNodeID):
"""
@param parentNodeID: The n-bit node ID of the node to which this
routing table belongs
@type parentNodeID: str
"""
def addContact(self, contact):
""" Add the given contact to the correct k-bucket; if it already
exists, its status will be updated
@param contact: The contact to add to this node's k-buckets
@type contact: kademlia.contact.Contact
"""
def findCloseNodes(self, key, count, _rpcNodeID=None):
""" Finds a number of known nodes closest to the node/value with the
specified key.
@param key: the n-bit key (i.e. the node or value ID) to search for
@type key: str
@param count: the amount of contacts to return
@type count: int
@param _rpcNodeID: Used during RPC, this is be the sender's Node ID
Whatever ID is passed in the parameter will get
excluded from the list of returned contacts.
@type _rpcNodeID: str
@return: A list of node contacts (C{kademlia.contact.Contact instances})
closest to the specified key.
This method will return C{k} (or C{count}, if specified)
contacts if at all possible; it will only return fewer if the
node is returning all of the contacts that it knows of.
@rtype: list
"""
def getContact(self, contactID):
""" Returns the (known) contact with the specified node ID
@raise ValueError: No contact with the specified contact ID is known
by this node
"""
def getRefreshList(self, startIndex=0, force=False):
""" Finds all k-buckets that need refreshing, starting at the
k-bucket with the specified index, and returns IDs to be searched for
in order to refresh those k-buckets
@param startIndex: The index of the bucket to start refreshing at;
this bucket and those further away from it will
be refreshed. For example, when joining the
network, this node will set this to the index of
the bucket after the one containing it's closest
neighbour.
@type startIndex: index
@param force: If this is C{True}, all buckets (in the specified range)
will be refreshed, regardless of the time they were last
accessed.
@type force: bool
@return: A list of node ID's that the parent node should search for
in order to refresh the routing Table
@rtype: list
"""
def removeContact(self, contactID):
""" Remove the contact with the specified node ID from the routing
table
@param contactID: The node ID of the contact to remove
@type contactID: str
"""
def touchKBucket(self, key):
""" Update the "last accessed" timestamp of the k-bucket which covers
the range containing the specified key in the key/ID space
@param key: A key in the range of the target k-bucket
@type key: str
"""
+230
View File
@@ -0,0 +1,230 @@
import logging
from twisted.internet import defer
from lbrynet.dht.distance import Distance
from lbrynet.dht.error import TimeoutError
from lbrynet.dht import constants
log = logging.getLogger(__name__)
def get_contact(contact_list, node_id, address, port):
for contact in contact_list:
if contact.id == node_id and contact.address == address and contact.port == port:
return contact
raise IndexError(node_id)
def expand_peer(compact_peer_info):
host = "{}.{}.{}.{}".format(*compact_peer_info[:4])
port = int.from_bytes(compact_peer_info[4:6], 'big')
peer_node_id = compact_peer_info[6:]
return (peer_node_id, host, port)
class _IterativeFind:
# TODO: use polymorphism to search for a value or node
# instead of using a find_value flag
def __init__(self, node, shortlist, key, rpc, exclude=None):
self.exclude = set(exclude or [])
self.node = node
self.finished_deferred = defer.Deferred()
# all distance operations in this class only care about the distance
# to self.key, so this makes it easier to calculate those
self.distance = Distance(key)
# The closest known and active node yet found
self.closest_node = None if not shortlist else shortlist[0]
self.prev_closest_node = None
# Shortlist of contact objects (the k closest known contacts to the key from the routing table)
self.shortlist = shortlist
# The search key
self.key = key
# The rpc method name (findValue or findNode)
self.rpc = rpc
# List of active queries; len() indicates number of active probes
self.active_probes = []
# List of contact (address, port) tuples that have already been queried, includes contacts that didn't reply
self.already_contacted = []
# A list of found and known-to-be-active remote nodes (Contact objects)
self.active_contacts = []
# Ensure only one searchIteration call is running at a time
self._search_iteration_semaphore = defer.DeferredSemaphore(1)
self._iteration_count = 0
self.find_value_result = {}
self.pending_iteration_calls = []
@property
def is_find_node_request(self):
return self.rpc == "findNode"
@property
def is_find_value_request(self):
return self.rpc == "findValue"
def is_closer(self, contact):
if not self.closest_node:
return True
return self.distance.is_closer(contact.id, self.closest_node.id)
def getContactTriples(self, result):
if self.is_find_value_request:
contact_triples = result[b'contacts']
else:
contact_triples = result
for contact_tup in contact_triples:
if not isinstance(contact_tup, (list, tuple)) or len(contact_tup) != 3:
raise ValueError("invalid contact triple")
contact_tup[1] = contact_tup[1].decode() # ips are strings
return contact_triples
def sortByDistance(self, contact_list):
"""Sort the list of contacts in order by distance from key"""
contact_list.sort(key=lambda c: self.distance(c.id))
def extendShortlist(self, contact, result):
# The "raw response" tuple contains the response message and the originating address info
originAddress = (contact.address, contact.port)
if self.finished_deferred.called:
return contact.id
if self.node.contact_manager.is_ignored(originAddress):
raise ValueError("contact is ignored")
if contact.id == self.node.node_id:
return contact.id
if contact not in self.active_contacts:
self.active_contacts.append(contact)
if contact not in self.shortlist:
self.shortlist.append(contact)
# Now grow extend the (unverified) shortlist with the returned contacts
# TODO: some validation on the result (for guarding against attacks)
# If we are looking for a value, first see if this result is the value
# we are looking for before treating it as a list of contact triples
if self.is_find_value_request and self.key in result:
# We have found the value
for peer in result[self.key]:
node_id, host, port = expand_peer(peer)
if (host, port) not in self.exclude:
self.find_value_result.setdefault(self.key, []).append((node_id, host, port))
if self.find_value_result:
self.finished_deferred.callback(self.find_value_result)
else:
if self.is_find_value_request:
# We are looking for a value, and the remote node didn't have it
# - mark it as the closest "empty" node, if it is
# TODO: store to this peer after finding the value as per the kademlia spec
if b'closestNodeNoValue' in self.find_value_result:
if self.is_closer(contact):
self.find_value_result[b'closestNodeNoValue'] = contact
else:
self.find_value_result[b'closestNodeNoValue'] = contact
contactTriples = self.getContactTriples(result)
for contactTriple in contactTriples:
if (contactTriple[1], contactTriple[2]) in ((c.address, c.port) for c in self.already_contacted):
continue
elif self.node.contact_manager.is_ignored((contactTriple[1], contactTriple[2])):
continue
else:
found_contact = self.node.contact_manager.make_contact(contactTriple[0], contactTriple[1],
contactTriple[2], self.node._protocol)
if found_contact not in self.shortlist:
self.shortlist.append(found_contact)
if not self.finished_deferred.called and self.should_stop():
self.sortByDistance(self.active_contacts)
self.finished_deferred.callback(self.active_contacts[:min(constants.k, len(self.active_contacts))])
return contact.id
@defer.inlineCallbacks
def probeContact(self, contact):
fn = getattr(contact, self.rpc)
try:
response = yield fn(self.key)
result = self.extendShortlist(contact, response)
defer.returnValue(result)
except (TimeoutError, defer.CancelledError, ValueError, IndexError):
defer.returnValue(contact.id)
def should_stop(self):
if self.is_find_value_request:
# search stops when it finds a value, let it run
return False
if self.prev_closest_node and self.closest_node and self.distance.is_closer(self.prev_closest_node.id,
self.closest_node.id):
# we're getting further away
return True
if len(self.active_contacts) >= constants.k:
# we have enough results
return True
return False
# Send parallel, asynchronous FIND_NODE RPCs to the shortlist of contacts
def _searchIteration(self):
# Sort the discovered active nodes from closest to furthest
if len(self.active_contacts):
self.sortByDistance(self.active_contacts)
self.prev_closest_node = self.closest_node
self.closest_node = self.active_contacts[0]
# Sort the current shortList before contacting other nodes
self.sortByDistance(self.shortlist)
probes = []
already_contacted_addresses = {(c.address, c.port) for c in self.already_contacted}
to_remove = []
for contact in self.shortlist:
if self.node.contact_manager.is_ignored((contact.address, contact.port)):
to_remove.append(contact) # a contact became bad during iteration
continue
if (contact.address, contact.port) not in already_contacted_addresses:
self.already_contacted.append(contact)
to_remove.append(contact)
probe = self.probeContact(contact)
probes.append(probe)
self.active_probes.append(probe)
if len(probes) == constants.alpha:
break
for contact in to_remove: # these contacts will be re-added to the shortlist when they reply successfully
self.shortlist.remove(contact)
# run the probes
if probes:
# Schedule the next iteration if there are any active
# calls (Kademlia uses loose parallelism)
self.searchIteration()
d = defer.DeferredList(probes, consumeErrors=True)
def _remove_probes(results):
for probe in probes:
self.active_probes.remove(probe)
return results
d.addCallback(_remove_probes)
elif not self.finished_deferred.called and not self.active_probes or self.should_stop():
# If no probes were sent, there will not be any improvement, so we're done
if self.is_find_value_request:
self.finished_deferred.callback(self.find_value_result)
else:
self.sortByDistance(self.active_contacts)
self.finished_deferred.callback(self.active_contacts[:min(constants.k, len(self.active_contacts))])
elif not self.finished_deferred.called:
# Force the next iteration
self.searchIteration()
def searchIteration(self, delay=constants.iterativeLookupDelay):
def _cancel_pending_iterations(result):
while self.pending_iteration_calls:
canceller = self.pending_iteration_calls.pop()
canceller()
return result
self.finished_deferred.addBoth(_cancel_pending_iterations)
self._iteration_count += 1
call, cancel = self.node.reactor_callLater(delay, self._search_iteration_semaphore.run, self._searchIteration)
self.pending_iteration_calls.append(cancel)
def iterativeFind(node, shortlist, key, rpc, exclude=None):
helper = _IterativeFind(node, shortlist, key, rpc, exclude)
helper.searchIteration(0)
return helper.finished_deferred
+147
View File
@@ -0,0 +1,147 @@
import logging
from lbrynet.dht import constants
from lbrynet.dht.distance import Distance
from lbrynet.dht.error import BucketFull
log = logging.getLogger(__name__)
class KBucket:
""" Description - later
"""
def __init__(self, rangeMin, rangeMax, node_id):
"""
@param rangeMin: The lower boundary for the range in the n-bit ID
space covered by this k-bucket
@param rangeMax: The upper boundary for the range in the ID space
covered by this k-bucket
"""
self.lastAccessed = 0
self.rangeMin = rangeMin
self.rangeMax = rangeMax
self._contacts = list()
self._node_id = node_id
def addContact(self, contact):
""" Add contact to _contact list in the right order. This will move the
contact to the end of the k-bucket if it is already present.
@raise kademlia.kbucket.BucketFull: Raised when the bucket is full and
the contact isn't in the bucket
already
@param contact: The contact to add
@type contact: dht.contact._Contact
"""
if contact in self._contacts:
# Move the existing contact to the end of the list
# - using the new contact to allow add-on data
# (e.g. optimization-specific stuff) to pe updated as well
self._contacts.remove(contact)
self._contacts.append(contact)
elif len(self._contacts) < constants.k:
self._contacts.append(contact)
else:
raise BucketFull("No space in bucket to insert contact")
def getContact(self, contactID):
"""Get the contact specified node ID
@raise IndexError: raised if the contact is not in the bucket
@param contactID: the node id of the contact to retrieve
@type contactID: str
@rtype: dht.contact._Contact
"""
for contact in self._contacts:
if contact.id == contactID:
return contact
raise IndexError(contactID)
def getContacts(self, count=-1, excludeContact=None, sort_distance_to=None):
""" Returns a list containing up to the first count number of contacts
@param count: The amount of contacts to return (if 0 or less, return
all contacts)
@type count: int
@param excludeContact: A node id to exclude; if this contact is in
the list of returned values, it will be
discarded before returning. If a C{str} is
passed as this argument, it must be the
contact's ID.
@type excludeContact: str
@param sort_distance_to: Sort distance to the id, defaulting to the parent node id. If False don't
sort the contacts
@raise IndexError: If the number of requested contacts is too large
@return: Return up to the first count number of contacts in a list
If no contacts are present an empty is returned
@rtype: list
"""
contacts = [contact for contact in self._contacts if contact.id != excludeContact]
# Return all contacts in bucket
if count <= 0:
count = len(contacts)
# Get current contact number
currentLen = len(contacts)
# If count greater than k - return only k contacts
if count > constants.k:
count = constants.k
if not currentLen:
return contacts
if sort_distance_to is False:
pass
else:
sort_distance_to = sort_distance_to or self._node_id
contacts.sort(key=lambda c: Distance(sort_distance_to)(c.id))
return contacts[:min(currentLen, count)]
def getBadOrUnknownContacts(self):
contacts = self.getContacts(sort_distance_to=False)
results = [contact for contact in contacts if contact.contact_is_good is False]
results.extend(contact for contact in contacts if contact.contact_is_good is None)
return results
def removeContact(self, contact):
""" Remove the contact from the bucket
@param contact: The contact to remove
@type contact: dht.contact._Contact
@raise ValueError: The specified contact is not in this bucket
"""
self._contacts.remove(contact)
def keyInRange(self, key):
""" Tests whether the specified key (i.e. node ID) is in the range
of the n-bit ID space covered by this k-bucket (in otherwords, it
returns whether or not the specified key should be placed in this
k-bucket)
@param key: The key to test
@type key: str or int
@return: C{True} if the key is in this k-bucket's range, or C{False}
if not.
@rtype: bool
"""
if isinstance(key, bytes):
key = int.from_bytes(key, 'big')
return self.rangeMin <= key < self.rangeMax
def __len__(self):
return len(self._contacts)
def __contains__(self, item):
return item in self._contacts
+90
View File
@@ -0,0 +1,90 @@
#!/usr/bin/env python
#
# This library is free software, distributed under the terms of
# the GNU Lesser General Public License Version 3, or any later version.
# See the COPYING file included in this archive
#
# The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net
from lbrynet.dht import msgtypes
class MessageTranslator:
""" Interface for RPC message translators/formatters
Classes inheriting from this should provide a translation services between
the classes used internally by this Kademlia implementation and the actual
data that is transmitted between nodes.
"""
def fromPrimitive(self, msgPrimitive):
""" Create an RPC Message from a message's string representation
@param msgPrimitive: The unencoded primitive representation of a message
@type msgPrimitive: str, int, list or dict
@return: The translated message object
@rtype: entangled.kademlia.msgtypes.Message
"""
def toPrimitive(self, message):
""" Create a string representation of a message
@param message: The message object
@type message: msgtypes.Message
@return: The message's primitive representation in a particular
messaging format
@rtype: str, int, list or dict
"""
class DefaultFormat(MessageTranslator):
""" The default on-the-wire message format for this library """
typeRequest, typeResponse, typeError = range(3)
headerType, headerMsgID, headerNodeID, headerPayload, headerArgs = range(5) # TODO: make str
@staticmethod
def get(primitive, key):
try:
return primitive[key]
except KeyError:
return primitive[str(key)] # TODO: switch to int()
def fromPrimitive(self, msgPrimitive):
msgType = self.get(msgPrimitive, self.headerType)
if msgType == self.typeRequest:
msg = msgtypes.RequestMessage(self.get(msgPrimitive, self.headerNodeID),
self.get(msgPrimitive, self.headerPayload),
self.get(msgPrimitive, self.headerArgs),
self.get(msgPrimitive, self.headerMsgID))
elif msgType == self.typeResponse:
msg = msgtypes.ResponseMessage(self.get(msgPrimitive, self.headerMsgID),
self.get(msgPrimitive, self.headerNodeID),
self.get(msgPrimitive, self.headerPayload))
elif msgType == self.typeError:
msg = msgtypes.ErrorMessage(self.get(msgPrimitive, self.headerMsgID),
self.get(msgPrimitive, self.headerNodeID),
self.get(msgPrimitive, self.headerPayload),
self.get(msgPrimitive, self.headerArgs))
else:
# Unknown message, no payload
msg = msgtypes.Message(msgPrimitive[self.headerMsgID], msgPrimitive[self.headerNodeID])
return msg
def toPrimitive(self, message):
msg = {self.headerMsgID: message.id,
self.headerNodeID: message.nodeID}
if isinstance(message, msgtypes.RequestMessage):
msg[self.headerType] = self.typeRequest
msg[self.headerPayload] = message.request
msg[self.headerArgs] = message.args
elif isinstance(message, msgtypes.ErrorMessage):
msg[self.headerType] = self.typeError
msg[self.headerPayload] = message.exceptionType
msg[self.headerArgs] = message.response
elif isinstance(message, msgtypes.ResponseMessage):
msg[self.headerType] = self.typeResponse
msg[self.headerPayload] = message.response
return msg
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python
#
# This library is free software, distributed under the terms of
# the GNU Lesser General Public License Version 3, or any later version.
# See the COPYING file included in this archive
#
# The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net
from lbrynet.utils import generate_id
from lbrynet.dht import constants
class Message:
""" Base class for messages - all "unknown" messages use this class """
def __init__(self, rpcID, nodeID):
if len(rpcID) != constants.rpc_id_length:
raise ValueError("invalid rpc id: %i bytes (expected 20)" % len(rpcID))
if len(nodeID) != constants.key_bits // 8:
raise ValueError("invalid node id: %i bytes (expected 48)" % len(nodeID))
self.id = rpcID
self.nodeID = nodeID
class RequestMessage(Message):
""" Message containing an RPC request """
def __init__(self, nodeID, method, methodArgs, rpcID=None):
if rpcID is None:
rpcID = generate_id()[:constants.rpc_id_length]
super().__init__(rpcID, nodeID)
self.request = method
self.args = methodArgs
class ResponseMessage(Message):
""" Message containing the result from a successful RPC request """
def __init__(self, rpcID, nodeID, response):
super().__init__(rpcID, nodeID)
self.response = response
class ErrorMessage(ResponseMessage):
""" Message containing the error from an unsuccessful RPC request """
def __init__(self, rpcID, nodeID, exceptionType, errorMessage):
super().__init__(rpcID, nodeID, errorMessage)
if isinstance(exceptionType, type):
exceptionType = (f'{exceptionType.__module__}.{exceptionType.__name__}').encode()
self.exceptionType = exceptionType
+625 -194
View File
@@ -1,218 +1,649 @@
import logging
import asyncio
import typing
import binascii
from lbrynet.utils import resolve_host
from lbrynet.dht import constants
from lbrynet.dht.protocol.distance import Distance
from lbrynet.dht.protocol.iterative_find import IterativeNodeFinder, IterativeValueFinder
from lbrynet.dht.protocol.protocol import KademliaProtocol
from lbrynet.dht.peer import KademliaPeer
import hashlib
import logging
from functools import reduce
if typing.TYPE_CHECKING:
from lbrynet.dht.peer import PeerManager
from twisted.internet import defer, error, task
from lbrynet.utils import generate_id, DeferredDict
from lbrynet.dht.call_later_manager import CallLaterManager
from lbrynet.dht.error import TimeoutError
from lbrynet.dht import constants, routingtable, datastore, protocol
from lbrynet.dht.contact import ContactManager
from lbrynet.dht.iterativefind import iterativeFind
log = logging.getLogger(__name__)
class Node:
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, udp_port: int,
internal_udp_port: int, peer_port: int, external_ip: str, rpc_timeout: float = constants.rpc_timeout,
split_buckets_under_index: int = constants.split_buckets_under_index):
self.loop = loop
self.internal_udp_port = internal_udp_port
self.protocol = KademliaProtocol(loop, peer_manager, node_id, external_ip, udp_port, peer_port, rpc_timeout,
split_buckets_under_index)
self.listening_port: asyncio.DatagramTransport = None
self.joined = asyncio.Event(loop=self.loop)
self._join_task: asyncio.Task = None
self._refresh_task: asyncio.Task = None
def rpcmethod(func):
""" Decorator to expose Node methods as remote procedure calls
async def refresh_node(self, force_once=False):
while True:
# remove peers with expired blob announcements from the datastore
self.protocol.data_store.removed_expired_peers()
Apply this decorator to methods in the Node class (or a subclass) in order
to make them remotely callable via the DHT's RPC mechanism.
"""
func.rpcmethod = True
return func
total_peers: typing.List['KademliaPeer'] = []
# add all peers in the routing table
total_peers.extend(self.protocol.routing_table.get_peers())
# add all the peers who have announed blobs to us
total_peers.extend(self.protocol.data_store.get_storing_contacts())
# get ids falling in the midpoint of each bucket that hasn't been recently updated
node_ids = self.protocol.routing_table.get_refresh_list(0, True)
# if we have 3 or fewer populated buckets get two random ids in the range of each to try and
# populate/split the buckets further
buckets_with_contacts = self.protocol.routing_table.buckets_with_contacts()
if buckets_with_contacts <= 3:
for i in range(buckets_with_contacts):
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
node_ids.append(self.protocol.routing_table.random_id_in_bucket_range(i))
class MockKademliaHelper:
def __init__(self, clock=None, callLater=None, resolve=None, listenUDP=None):
if not listenUDP or not resolve or not callLater or not clock:
from twisted.internet import reactor
listenUDP = listenUDP or reactor.listenUDP
resolve = resolve or reactor.resolve
callLater = callLater or reactor.callLater
clock = clock or reactor
if self.protocol.routing_table.get_peers():
# if we have node ids to look up, perform the iterative search until we have k results
while node_ids:
peers = await self.peer_search(node_ids.pop())
total_peers.extend(peers)
self.clock = clock
self.contact_manager = ContactManager(self.clock.seconds)
self.reactor_listenUDP = listenUDP
self.reactor_resolve = resolve
self.call_later_manager = CallLaterManager(callLater)
self.reactor_callLater = self.call_later_manager.call_later
self.reactor_callSoon = self.call_later_manager.call_soon
self._listeningPort = None # object implementing Twisted
# IListeningPort This will contain a deferred created when
# joining the network, to enable publishing/retrieving
# information from the DHT as soon as the node is part of the
# network (add callbacks to this deferred if scheduling such
# operations before the node has finished joining the network)
def get_looping_call(self, fn, *args, **kwargs):
lc = task.LoopingCall(fn, *args, **kwargs)
lc.clock = self.clock
return lc
def safe_stop_looping_call(self, lc):
if lc and lc.running:
return lc.stop()
return defer.succeed(None)
def safe_start_looping_call(self, lc, t):
if lc and not lc.running:
lc.start(t)
class Node(MockKademliaHelper):
""" Local node in the Kademlia network
This class represents a single local node in a Kademlia network; in other
words, this class encapsulates an Entangled-using application's "presence"
in a Kademlia network.
In Entangled, all interactions with the Kademlia network by a client
application is performed via this class (or a subclass).
"""
def __init__(self, node_id=None, udpPort=4000, dataStore=None,
routingTableClass=None, networkProtocol=None,
externalIP=None, peerPort=3333, listenUDP=None,
callLater=None, resolve=None, clock=None,
interface='', externalUDPPort=None):
"""
@param dataStore: The data store to use. This must be class inheriting
from the C{DataStore} interface (or providing the
same API). How the data store manages its data
internally is up to the implementation of that data
store.
@type dataStore: entangled.kademlia.datastore.DataStore
@param routingTable: The routing table class to use. Since there exists
some ambiguity as to how the routing table should be
implemented in Kademlia, a different routing table
may be used, as long as the appropriate API is
exposed. This should be a class, not an object,
in order to allow the Node to pass an
auto-generated node ID to the routingtable object
upon instantiation (if necessary).
@type routingTable: entangled.kademlia.routingtable.RoutingTable
@param networkProtocol: The network protocol to use. This can be
overridden from the default to (for example)
change the format of the physical RPC messages
being transmitted.
@type networkProtocol: entangled.kademlia.protocol.KademliaProtocol
@param externalIP: the IP at which this node can be contacted
@param peerPort: the port at which this node announces it has a blob for
"""
super().__init__(clock, callLater, resolve, listenUDP)
self.node_id = node_id or self._generateID()
self.port = udpPort
self._listen_interface = interface
self._change_token_lc = self.get_looping_call(self.change_token)
self._refresh_node_lc = self.get_looping_call(self._refreshNode)
self._refresh_contacts_lc = self.get_looping_call(self._refreshContacts)
# Create k-buckets (for storing contacts)
if routingTableClass is None:
self._routingTable = routingtable.TreeRoutingTable(self.node_id, self.clock.seconds)
else:
self._routingTable = routingTableClass(self.node_id, self.clock.seconds)
self._protocol = networkProtocol or protocol.KademliaProtocol(self)
self.token_secret = self._generateID()
self.old_token_secret = None
self.externalIP = externalIP
self.peerPort = peerPort
self.externalUDPPort = externalUDPPort or self.port
self._dataStore = dataStore or datastore.DictDataStore(self.clock.seconds)
self._join_deferred = None
#def __del__(self):
# log.warning("unclean shutdown of the dht node")
# if hasattr(self, "_listeningPort") and self._listeningPort is not None:
# self._listeningPort.stopListening()
def __str__(self):
return '<%s.%s object; ID: %s, IP address: %s, UDP port: %d>' % (
self.__module__, self.__class__.__name__, binascii.hexlify(self.node_id), self.externalIP, self.port)
@defer.inlineCallbacks
def stop(self):
# stop LoopingCalls:
yield self.safe_stop_looping_call(self._refresh_node_lc)
yield self.safe_stop_looping_call(self._change_token_lc)
yield self.safe_stop_looping_call(self._refresh_contacts_lc)
if self._listeningPort is not None:
yield self._listeningPort.stopListening()
self._listeningPort = None
def start_listening(self):
if not self._listeningPort:
try:
self._listeningPort = self.reactor_listenUDP(self.port, self._protocol,
interface=self._listen_interface)
except error.CannotListenError as e:
import traceback
log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc())
raise ValueError("%s lbrynet may already be running." % str(e))
else:
log.warning("Already bound to port %s", self._listeningPort)
@defer.inlineCallbacks
def joinNetwork(self, known_node_addresses=(('jack.lbry.tech', 4455), )):
"""
Attempt to join the dht, retry every 30 seconds if unsuccessful
:param known_node_addresses: [(str, int)] list of hostnames and ports for known dht seed nodes
"""
self._join_deferred = defer.Deferred()
known_node_resolution = {}
@defer.inlineCallbacks
def _resolve_seeds():
result = {}
for host, port in known_node_addresses:
node_address = yield self.reactor_resolve(host)
result[(host, port)] = node_address
defer.returnValue(result)
if not known_node_resolution:
known_node_resolution = yield _resolve_seeds()
# we are one of the seed nodes, don't add ourselves
if (self.externalIP, self.port) in known_node_resolution.values():
del known_node_resolution[(self.externalIP, self.port)]
known_node_addresses.remove((self.externalIP, self.port))
def _ping_contacts(contacts):
d = DeferredDict({contact: contact.ping() for contact in contacts}, consumeErrors=True)
d.addErrback(lambda err: err.trap(TimeoutError))
return d
@defer.inlineCallbacks
def _initialize_routing():
bootstrap_contacts = []
contact_addresses = {(c.address, c.port): c for c in self.contacts}
for (host, port), ip_address in known_node_resolution.items():
if (host, port) not in contact_addresses:
# Create temporary contact information for the list of addresses of known nodes
# The contact node id will be set with the responding node id when we initialize it to None
contact = self.contact_manager.make_contact(None, ip_address, port, self._protocol)
bootstrap_contacts.append(contact)
else:
for contact in self.contacts:
if contact.address == ip_address and contact.port == port:
if not contact.id:
bootstrap_contacts.append(contact)
break
if not bootstrap_contacts:
log.warning("no bootstrap contacts to ping")
ping_result = yield _ping_contacts(bootstrap_contacts)
shortlist = list(ping_result.keys())
if not shortlist:
log.warning("failed to ping %i bootstrap contacts", len(bootstrap_contacts))
defer.returnValue(None)
else:
if force_once:
break
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval // 4, fut.set_result, None)
await fut
continue
# find the closest peers to us
closest = yield self._iterativeFind(self.node_id, shortlist if not self.contacts else None)
yield _ping_contacts(closest)
# query random hashes in our bucket key ranges to fill or split them
random_ids_in_range = self._routingTable.getRefreshList()
while random_ids_in_range:
yield self.iterativeFindNode(random_ids_in_range.pop())
defer.returnValue(None)
# ping the set of peers; upon success/failure the routing able and last replied/failed time will be updated
to_ping = [peer for peer in set(total_peers) if self.protocol.peer_manager.peer_is_good(peer) is not True]
if to_ping:
self.protocol.ping_queue.enqueue_maybe_ping(*to_ping, delay=0)
if force_once:
break
@defer.inlineCallbacks
def _iterative_join(joined_d=None, last_buckets_with_contacts=None):
log.info("Attempting to join the DHT network, %i contacts known so far", len(self.contacts))
joined_d = joined_d or defer.Deferred()
yield _initialize_routing()
buckets_with_contacts = self.bucketsWithContacts()
if last_buckets_with_contacts and last_buckets_with_contacts == buckets_with_contacts:
if not joined_d.called:
joined_d.callback(True)
elif buckets_with_contacts < 4:
self.reactor_callLater(0, _iterative_join, joined_d, buckets_with_contacts)
elif not joined_d.called:
joined_d.callback(None)
yield joined_d
if not self._join_deferred.called:
self._join_deferred.callback(True)
defer.returnValue(None)
fut = asyncio.Future(loop=self.loop)
self.loop.call_later(constants.refresh_interval, fut.set_result, None)
await fut
yield _iterative_join()
async def announce_blob(self, blob_hash: str) -> typing.List[bytes]:
hash_value = binascii.unhexlify(blob_hash.encode())
assert len(hash_value) == constants.hash_length
peers = await self.peer_search(hash_value)
@defer.inlineCallbacks
def start(self, known_node_addresses=None, block_on_join=False):
""" Causes the Node to attempt to join the DHT network by contacting the
known DHT nodes. This can be called multiple times if the previous attempt
has failed or if the Node has lost all the contacts.
if not self.protocol.external_ip:
raise Exception("Cannot determine external IP")
log.debug("Store to %i peers", len(peers))
for peer in peers:
log.debug("store to %s %s %s", peer.address, peer.udp_port, peer.tcp_port)
stored_to_tup = await asyncio.gather(
*(self.protocol.store_to_peer(hash_value, peer) for peer in peers), loop=self.loop
)
stored_to = [node_id for node_id, contacted in stored_to_tup if contacted]
if stored_to:
log.info("Stored %s to %i of %i attempted peers", binascii.hexlify(hash_value).decode()[:8],
len(stored_to), len(peers))
else:
log.warning("Failed announcing %s, stored to 0 peers", blob_hash[:8])
return stored_to
@param known_node_addresses: A sequence of tuples containing IP address
information for existing nodes on the
Kademlia network, in the format:
C{(<ip address>, (udp port>)}
@type known_node_addresses: list
"""
def stop(self) -> None:
if self.joined.is_set():
self.joined.clear()
if self._join_task:
self._join_task.cancel()
if self._refresh_task and not (self._refresh_task.done() or self._refresh_task.cancelled()):
self._refresh_task.cancel()
if self.protocol and self.protocol.ping_queue.running:
self.protocol.ping_queue.stop()
self.protocol.stop()
if self.listening_port is not None:
self.listening_port.close()
self._join_task = None
self.listening_port = None
log.info("Stopped DHT node")
self.start_listening()
yield self._protocol._listening
# TODO: Refresh all k-buckets further away than this node's closest neighbour
d = self.joinNetwork(known_node_addresses or [])
d.addCallback(lambda _: self.start_looping_calls())
d.addCallback(lambda _: log.info("Joined the dht"))
if block_on_join:
yield d
async def start_listening(self, interface: str = '') -> None:
if not self.listening_port:
self.listening_port, _ = await self.loop.create_datagram_endpoint(
lambda: self.protocol, (interface, self.internal_udp_port)
)
log.info("DHT node listening on UDP %s:%i", interface, self.internal_udp_port)
self.protocol.start()
else:
log.warning("Already bound to port %s", self.listening_port)
def start_looping_calls(self):
self.safe_start_looping_call(self._change_token_lc, constants.tokenSecretChangeInterval)
# Start refreshing k-buckets periodically, if necessary
self.safe_start_looping_call(self._refresh_node_lc, constants.checkRefreshInterval)
self.safe_start_looping_call(self._refresh_contacts_lc, 60)
async def join_network(self, interface: typing.Optional[str] = '',
known_node_urls: typing.Optional[typing.List[typing.Tuple[str, int]]] = None):
if not self.listening_port:
await self.start_listening(interface)
self.protocol.ping_queue.start()
self._refresh_task = self.loop.create_task(self.refresh_node())
@property
def contacts(self):
def _inner():
for i in range(len(self._routingTable._buckets)):
for contact in self._routingTable._buckets[i]._contacts:
yield contact
return list(_inner())
# resolve the known node urls
known_node_addresses = []
url_to_addr = {}
def hasContacts(self):
for bucket in self._routingTable._buckets:
if bucket._contacts:
return True
return False
if known_node_urls:
for host, port in known_node_urls:
address = await resolve_host(host, port, proto='udp')
if (address, port) not in known_node_addresses and\
(address, port) != (self.protocol.external_ip, self.protocol.udp_port):
known_node_addresses.append((address, port))
url_to_addr[address] = host
def bucketsWithContacts(self):
return self._routingTable.bucketsWithContacts()
if known_node_addresses:
peers = [
KademliaPeer(self.loop, address, udp_port=port)
for (address, port) in known_node_addresses
]
while True:
if not self.protocol.routing_table.get_peers():
if self.joined.is_set():
self.joined.clear()
self.protocol.peer_manager.reset()
self.protocol.ping_queue.enqueue_maybe_ping(*peers, delay=0.0)
peers.extend(await self.peer_search(self.protocol.node_id, shortlist=peers, count=32))
if self.protocol.routing_table.get_peers():
self.joined.set()
log.info(
"Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts())
else:
continue
await asyncio.sleep(1, loop=self.loop)
log.info("Joined DHT, %i peers known in %i buckets", len(self.protocol.routing_table.get_peers()),
self.protocol.routing_table.buckets_with_contacts())
self.joined.set()
def start(self, interface: str, known_node_urls: typing.List[typing.Tuple[str, int]]):
self._join_task = self.loop.create_task(
self.join_network(
interface=interface, known_node_urls=known_node_urls
)
)
def get_iterative_node_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
bottom_out_limit: int = constants.bottom_out_limit,
max_results: int = constants.k) -> IterativeNodeFinder:
return IterativeNodeFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
key, bottom_out_limit, max_results, None, shortlist)
def get_iterative_value_finder(self, key: bytes, shortlist: typing.Optional[typing.List['KademliaPeer']] = None,
bottom_out_limit: int = 40,
max_results: int = -1) -> IterativeValueFinder:
return IterativeValueFinder(self.loop, self.protocol.peer_manager, self.protocol.routing_table, self.protocol,
key, bottom_out_limit, max_results, None, shortlist)
async def peer_search(self, node_id: bytes, count=constants.k, max_results=constants.k*2,
bottom_out_limit=20, shortlist: typing.Optional[typing.List['KademliaPeer']] = None
) -> typing.List['KademliaPeer']:
peers = []
async for iteration_peers in self.get_iterative_node_finder(
node_id, shortlist=shortlist, bottom_out_limit=bottom_out_limit, max_results=max_results):
peers.extend(iteration_peers)
distance = Distance(node_id)
peers.sort(key=lambda peer: distance(peer.node_id))
return peers[:count]
async def _accumulate_search_junction(self, search_queue: asyncio.Queue,
result_queue: asyncio.Queue):
tasks = []
@defer.inlineCallbacks
def storeToContact(self, blob_hash, contact):
try:
while True:
blob_hash = await search_queue.get()
tasks.append(self.loop.create_task(self._value_producer(blob_hash, result_queue)))
finally:
for task in tasks:
task.cancel()
if not contact.token:
yield contact.findValue(blob_hash)
res = yield contact.store(blob_hash, contact.token, self.peerPort, self.node_id, 0)
if res != b"OK":
raise ValueError(res)
log.debug("Stored %s to %s (%s)", binascii.hexlify(blob_hash), contact.log_id(), contact.address)
return True
except protocol.TimeoutError:
log.debug("Timeout while storing blob_hash %s at %s",
binascii.hexlify(blob_hash), contact.log_id())
except ValueError as err:
log.error("Unexpected response: %s" % err)
except Exception as err:
if 'Invalid token' in str(err):
contact.update_token(None)
log.error("Unexpected error while storing blob_hash %s at %s: %s",
binascii.hexlify(blob_hash), contact, err)
return False
async def _value_producer(self, blob_hash: str, result_queue: asyncio.Queue):
async for results in self.get_iterative_value_finder(binascii.unhexlify(blob_hash.encode())):
result_queue.put_nowait(results)
@defer.inlineCallbacks
def announceHaveBlob(self, blob_hash):
contacts = yield self.iterativeFindNode(blob_hash)
def accumulate_peers(self, search_queue: asyncio.Queue,
peer_queue: typing.Optional[asyncio.Queue] = None) -> typing.Tuple[
asyncio.Queue, asyncio.Task]:
q = peer_queue or asyncio.Queue(loop=self.loop)
return q, self.loop.create_task(self._accumulate_search_junction(search_queue, q))
if not self.externalIP:
raise Exception("Cannot determine external IP: %s" % self.externalIP)
stored_to = yield DeferredDict({contact: self.storeToContact(blob_hash, contact) for contact in contacts})
contacted_node_ids = [binascii.hexlify(contact.id) for contact in stored_to.keys() if stored_to[contact]]
log.debug("Stored %s to %i of %i attempted peers", binascii.hexlify(blob_hash),
len(contacted_node_ids), len(contacts))
defer.returnValue(contacted_node_ids)
def change_token(self):
self.old_token_secret = self.token_secret
self.token_secret = self._generateID()
def make_token(self, compact_ip):
h = hashlib.new('sha384')
h.update(self.token_secret + compact_ip)
return h.digest()
def verify_token(self, token, compact_ip):
h = hashlib.new('sha384')
h.update(self.token_secret + compact_ip)
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
h = hashlib.new('sha384')
h.update(self.old_token_secret + compact_ip)
if not token == h.digest():
return False
return True
def iterativeFindNode(self, key):
""" The basic Kademlia node lookup operation
Call this to find a remote node in the P2P overlay network.
@param key: the n-bit key (i.e. the node or value ID) to search for
@type key: str
@return: This immediately returns a deferred object, which will return
a list of k "closest" contacts (C{kademlia.contact.Contact}
objects) to the specified key as soon as the operation is
finished.
@rtype: twisted.internet.defer.Deferred
"""
return self._iterativeFind(key)
@defer.inlineCallbacks
def iterativeFindValue(self, key, exclude=None):
""" The Kademlia search operation (deterministic)
Call this to retrieve data from the DHT.
@param key: the n-bit key (i.e. the value ID) to search for
@type key: str
@return: This immediately returns a deferred object, which will return
either one of two things:
- If the value was found, it will return a Python
dictionary containing the searched-for key (the C{key}
parameter passed to this method), and its associated
value, in the format:
C{<str>key: <str>data_value}
- If the value was not found, it will return a list of k
"closest" contacts (C{kademlia.contact.Contact} objects)
to the specified key
@rtype: twisted.internet.defer.Deferred
"""
if len(key) != constants.key_bits // 8:
raise ValueError("invalid key length!")
# Execute the search
find_result = yield self._iterativeFind(key, rpc='findValue', exclude=exclude)
if isinstance(find_result, dict):
# We have found the value; now see who was the closest contact without it...
# ...and store the key/value pair
pass
else:
# The value wasn't found, but a list of contacts was returned
# Now, see if we have the value (it might seem wasteful to search on the network
# first, but it ensures that all values are properly propagated through the
# network
if self._dataStore.hasPeersForBlob(key):
# Ok, we have the value locally, so use that
# Send this value to the closest node without it
peers = self._dataStore.getPeersForBlob(key)
find_result = {key: peers}
else:
pass
defer.returnValue(list(set(find_result.get(key, []) if find_result else [])))
# TODO: get this working
# if 'closestNodeNoValue' in find_result:
# closest_node_without_value = find_result['closestNodeNoValue']
# try:
# response, address = yield closest_node_without_value.findValue(key, rawResponse=True)
# yield closest_node_without_value.store(key, response.response['token'], self.peerPort)
# except TimeoutError:
# pass
def addContact(self, contact):
""" Add/update the given contact; simple wrapper for the same method
in this object's RoutingTable object
@param contact: The contact to add to this node's k-buckets
@type contact: kademlia.contact.Contact
"""
return self._routingTable.addContact(contact)
def removeContact(self, contact):
""" Remove the contact with the specified node ID from this node's
table of known nodes. This is a simple wrapper for the same method
in this object's RoutingTable object
@param contact: The Contact object to remove
@type contact: _Contact
"""
self._routingTable.removeContact(contact)
def findContact(self, contactID):
""" Find a entangled.kademlia.contact.Contact object for the specified
cotact ID
@param contactID: The contact ID of the required Contact object
@type contactID: str
@return: Contact object of remote node with the specified node ID,
or None if the contact was not found
@rtype: twisted.internet.defer.Deferred
"""
try:
df = defer.succeed(self._routingTable.getContact(contactID))
except (ValueError, IndexError):
df = self.iterativeFindNode(contactID)
df.addCallback(lambda nodes: ([node for node in nodes if node.id == contactID] or (None,))[0])
return df
@rpcmethod
def ping(self):
""" Used to verify contact between two Kademlia nodes
@rtype: str
"""
return b'pong'
@rpcmethod
def store(self, rpc_contact, blob_hash, token, port, originalPublisherID, age):
""" Store the received data in this node's local datastore
@param blob_hash: The hash of the data
@type blob_hash: str
@param token: The token we previously returned when this contact sent us a findValue
@type token: str
@param port: The TCP port the contact is listening on for requests for this blob (the peerPort)
@type port: int
@param originalPublisherID: The node ID of the node that is the publisher of the data
@type originalPublisherID: str
@param age: The relative age of the data (time in seconds since it was
originally published). Note that the original publish time
isn't actually given, to compensate for clock skew between
different nodes.
@type age: int
@rtype: str
"""
if originalPublisherID is None:
originalPublisherID = rpc_contact.id
compact_ip = rpc_contact.compact_ip()
if self.clock.seconds() - self._protocol.started_listening_time < constants.tokenSecretChangeInterval:
pass
elif not self.verify_token(token, compact_ip):
raise ValueError("Invalid token")
if 0 <= port <= 65536:
compact_port = port.to_bytes(2, 'big')
else:
raise TypeError(f'Invalid port: {port}')
compact_address = compact_ip + compact_port + rpc_contact.id
now = int(self.clock.seconds())
originallyPublished = now - age
self._dataStore.addPeerToBlob(rpc_contact, blob_hash, compact_address, now, originallyPublished,
originalPublisherID)
return b'OK'
@rpcmethod
def findNode(self, rpc_contact, key):
""" Finds a number of known nodes closest to the node/value with the
specified key.
@param key: the n-bit key (i.e. the node or value ID) to search for
@type key: str
@return: A list of contact triples closest to the specified key.
This method will return C{k} (or C{count}, if specified)
contacts if at all possible; it will only return fewer if the
node is returning all of the contacts that it knows of.
@rtype: list
"""
if len(key) != constants.key_bits // 8:
raise ValueError("invalid contact id length: %i" % len(key))
contacts = self._routingTable.findCloseNodes(key, sender_node_id=rpc_contact.id)
contact_triples = []
for contact in contacts:
contact_triples.append((contact.id, contact.address, contact.port))
return contact_triples
@rpcmethod
def findValue(self, rpc_contact, key):
""" Return the value associated with the specified key if present in
this node's data, otherwise execute FIND_NODE for the key
@param key: The hashtable key of the data to return
@type key: str
@return: A dictionary containing the requested key/value pair,
or a list of contact triples closest to the requested key.
@rtype: dict or list
"""
if len(key) != constants.key_bits // 8:
raise ValueError("invalid blob hash length: %i" % len(key))
response = {
b'token': self.make_token(rpc_contact.compact_ip()),
}
if self._protocol._protocolVersion:
response[b'protocolVersion'] = self._protocol._protocolVersion
# get peers we have stored for this blob
has_other_peers = self._dataStore.hasPeersForBlob(key)
peers = []
if has_other_peers:
peers.extend(self._dataStore.getPeersForBlob(key))
# if we don't have k storing peers to return and we have this hash locally, include our contact information
if len(peers) < constants.k and key in self._dataStore.completed_blobs:
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), self.externalIP.split('.'), bytearray())
compact_port = self.peerPort.to_bytes(2, 'big')
compact_address = compact_ip + compact_port + self.node_id
peers.append(compact_address)
if peers:
response[key] = peers
else:
response[b'contacts'] = self.findNode(rpc_contact, key)
return response
def _generateID(self):
""" Generates an n-bit pseudo-random identifier
@return: A globally unique n-bit pseudo-random identifier
@rtype: str
"""
return generate_id()
# from lbrynet.p2p.utils import profile_deferred
# @profile_deferred()
@defer.inlineCallbacks
def _iterativeFind(self, key, startupShortlist=None, rpc='findNode', exclude=None):
""" The basic Kademlia iterative lookup operation (for nodes/values)
This builds a list of k "closest" contacts through iterative use of
the "FIND_NODE" RPC, or if C{findValue} is set to C{True}, using the
"FIND_VALUE" RPC, in which case the value (if found) may be returned
instead of a list of contacts
@param key: the n-bit key (i.e. the node or value ID) to search for
@type key: str
@param startupShortlist: A list of contacts to use as the starting
shortlist for this search; this is normally
only used when the node joins the network
@type startupShortlist: list
@param rpc: The name of the RPC to issue to remote nodes during the
Kademlia lookup operation (e.g. this sets whether this
algorithm should search for a data value (if
rpc='findValue') or not. It can thus be used to perform
other operations that piggy-back on the basic Kademlia
lookup operation (Entangled's "delete" RPC, for instance).
@type rpc: str
@return: If C{findValue} is C{True}, the algorithm will stop as soon
as a data value for C{key} is found, and return a dictionary
containing the key and the found value. Otherwise, it will
return a list of the k closest nodes to the specified key
@rtype: twisted.internet.defer.Deferred
"""
if len(key) != constants.key_bits // 8:
raise ValueError("invalid key length: %i" % len(key))
if startupShortlist is None:
shortlist = self._routingTable.findCloseNodes(key)
# if key != self.node_id:
# # Update the "last accessed" timestamp for the appropriate k-bucket
# self._routingTable.touchKBucket(key)
if len(shortlist) == 0:
log.warning("This node doesn't know any other nodes")
# This node doesn't know of any other nodes
fakeDf = defer.Deferred()
fakeDf.callback([])
result = yield fakeDf
defer.returnValue(result)
else:
# This is used during the bootstrap process
shortlist = startupShortlist
result = yield iterativeFind(self, shortlist, key, rpc, exclude=exclude)
defer.returnValue(result)
@defer.inlineCallbacks
def _refreshNode(self):
""" Periodically called to perform k-bucket refreshes and data
replication/republishing as necessary """
yield self._refreshRoutingTable()
self._dataStore.removeExpiredPeers()
self._refreshStoringPeers()
defer.returnValue(None)
def _refreshContacts(self):
self._protocol._ping_queue.enqueue_maybe_ping(*self.contacts, delay=0)
def _refreshStoringPeers(self):
self._protocol._ping_queue.enqueue_maybe_ping(*self._dataStore.getStoringContacts(), delay=0)
@defer.inlineCallbacks
def _refreshRoutingTable(self):
nodeIDs = self._routingTable.getRefreshList(0, False)
while nodeIDs:
searchID = nodeIDs.pop()
yield self.iterativeFindNode(searchID)
defer.returnValue(None)
-198
View File
@@ -1,198 +0,0 @@
import typing
import asyncio
import logging
import ipaddress
from binascii import hexlify
from functools import lru_cache
from lbrynet.dht import constants
from lbrynet.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
log = logging.getLogger(__name__)
def is_valid_ipv4(address):
try:
ip = ipaddress.ip_address(address)
return ip.version == 4
except ipaddress.AddressValueError:
return False
class PeerManager:
def __init__(self, loop: asyncio.BaseEventLoop):
self._loop = loop
self._rpc_failures: typing.Dict[
typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]]
] = {}
self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {}
self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {}
self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {}
self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {}
self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {}
self._node_tokens: typing.Dict[bytes, (float, bytes)] = {}
def reset(self):
for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested):
statistic.clear()
def report_failure(self, address: str, udp_port: int):
now = self._loop.time()
_, previous = self._rpc_failures.pop((address, udp_port), (None, None))
self._rpc_failures[(address, udp_port)] = (previous, now)
def report_last_sent(self, address: str, udp_port: int):
now = self._loop.time()
self._last_sent[(address, udp_port)] = now
def report_last_replied(self, address: str, udp_port: int):
now = self._loop.time()
self._last_replied[(address, udp_port)] = now
def report_last_requested(self, address: str, udp_port: int):
now = self._loop.time()
self._last_requested[(address, udp_port)] = now
def clear_token(self, node_id: bytes):
self._node_tokens.pop(node_id, None)
def update_token(self, node_id: bytes, token: bytes):
now = self._loop.time()
self._node_tokens[node_id] = (now, token)
def get_node_token(self, node_id: bytes) -> typing.Optional[bytes]:
ts, token = self._node_tokens.get(node_id, (None, None))
if ts and ts > self._loop.time() - constants.token_secret_refresh_interval:
return token
def get_last_replied(self, address: str, udp_port: int) -> typing.Optional[float]:
return self._last_replied.get((address, udp_port))
def update_contact_triple(self, node_id: bytes, address: str, udp_port: int):
"""
Update the mapping of node_id -> address tuple and that of address tuple -> node_id
This is to handle peers changing addresses and ids while assuring that the we only ever have
one node id / address tuple mapped to each other
"""
if (address, udp_port) in self._node_id_mapping:
self._node_id_reverse_mapping.pop(self._node_id_mapping.pop((address, udp_port)))
if node_id in self._node_id_reverse_mapping:
self._node_id_mapping.pop(self._node_id_reverse_mapping.pop(node_id))
self._node_id_mapping[(address, udp_port)] = node_id
self._node_id_reverse_mapping[node_id] = (address, udp_port)
@lru_cache(maxsize=400)
def get_kademlia_peer(self, node_id: bytes, address: str, udp_port: int) -> 'KademliaPeer':
return KademliaPeer(self._loop, address, node_id, udp_port)
def prune(self): # TODO: periodically call this
now = self._loop.time()
to_pop = []
for (address, udp_port), (_, last_failure) in self._rpc_failures.items():
if last_failure and last_failure < now - constants.rpc_attempts_pruning_window:
to_pop.append((address, udp_port))
while to_pop:
del self._rpc_failures[to_pop.pop()]
to_pop = []
for node_id, (age, token) in self._node_tokens.items():
if age < now - constants.token_secret_refresh_interval:
to_pop.append(node_id)
while to_pop:
del self._node_tokens[to_pop.pop()]
def contact_triple_is_good(self, node_id: bytes, address: str, udp_port: int):
"""
:return: False if peer is bad, None if peer is unknown, or True if peer is good
"""
delay = self._loop.time() - constants.check_refresh_interval
# fixme: find a way to re-enable that without breaking other parts
#if node_id not in self._node_id_reverse_mapping or (address, udp_port) not in self._node_id_mapping:
# return
#addr_tup = (address, udp_port)
#if self._node_id_reverse_mapping[node_id] != addr_tup or self._node_id_mapping[addr_tup] != node_id:
# return
previous_failure, most_recent_failure = self._rpc_failures.get((address, udp_port), (None, None))
last_requested = self._last_requested.get((address, udp_port))
last_replied = self._last_replied.get((address, udp_port))
if most_recent_failure and last_replied:
if delay < last_replied > most_recent_failure:
return True
elif last_replied > most_recent_failure:
return
return False
elif previous_failure and most_recent_failure and most_recent_failure > delay:
return False
elif last_replied and last_replied > delay:
return True
elif last_requested and last_requested > delay:
return None
return
def peer_is_good(self, peer: 'KademliaPeer'):
return self.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port)
def decode_tcp_peer_from_compact_address(self, compact_address: bytes) -> 'KademliaPeer':
node_id, address, tcp_port = decode_compact_address(compact_address)
return KademliaPeer(self._loop, address, node_id, tcp_port=tcp_port)
class KademliaPeer:
__slots__ = [
'loop',
'_node_id',
'address',
'udp_port',
'tcp_port',
'protocol_version',
]
def __init__(self, loop: asyncio.BaseEventLoop, address: str, node_id: typing.Optional[bytes] = None,
udp_port: typing.Optional[int] = None, tcp_port: typing.Optional[int] = None):
if node_id is not None:
if not len(node_id) == constants.hash_length:
raise ValueError("invalid node_id: {}".format(hexlify(node_id).decode()))
if udp_port is not None and not 0 <= udp_port <= 65536:
raise ValueError("invalid udp port")
if tcp_port and not 0 <= tcp_port <= 65536:
raise ValueError("invalid tcp port")
if not is_valid_ipv4(address):
raise ValueError("invalid ip address")
self.loop = loop
self._node_id = node_id
self.address = address
self.udp_port = udp_port
self.tcp_port = tcp_port
self.protocol_version = 1
def update_tcp_port(self, tcp_port: int):
self.tcp_port = tcp_port
def update_udp_port(self, udp_port: int):
self.udp_port = udp_port
def set_id(self, node_id):
if not self._node_id:
self._node_id = node_id
@property
def node_id(self) -> bytes:
return self._node_id
def compact_address_udp(self) -> bytearray:
return make_compact_address(self.node_id, self.address, self.udp_port)
def compact_address_tcp(self) -> bytearray:
return make_compact_address(self.node_id, self.address, self.tcp_port)
def compact_ip(self):
return make_compact_ip(self.address)
def __eq__(self, other):
if not isinstance(other, KademliaPeer):
raise TypeError("invalid type to compare with Peer: %s" % str(type(other)))
return (self.node_id, self.address, self.udp_port) == (other.node_id, other.address, other.udp_port)
def __hash__(self):
return hash((self.node_id, self.address, self.udp_port))
+493
View File
@@ -0,0 +1,493 @@
import logging
import errno
from binascii import hexlify
from twisted.internet import protocol, defer
from lbrynet.dht import constants, encoding, msgformat, msgtypes
from lbrynet.dht.error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
log = logging.getLogger(__name__)
class PingQueue:
"""
Schedules a 15 minute delayed ping after a new node sends us a query. This is so the new node gets added to the
routing table after having been given enough time for a pinhole to expire.
"""
def __init__(self, node):
self._node = node
self._enqueued_contacts = {}
self._pending_contacts = {}
self._process_lc = node.get_looping_call(self._process)
def enqueue_maybe_ping(self, *contacts, **kwargs):
delay = kwargs.get('delay', constants.checkRefreshInterval)
no_op = (defer.succeed(None), lambda: None)
for contact in contacts:
if delay and contact not in self._enqueued_contacts:
self._pending_contacts.setdefault(contact, self._node.clock.seconds() + delay)
else:
self._enqueued_contacts.setdefault(contact, no_op)
@defer.inlineCallbacks
def _ping(self, contact):
if contact.contact_is_good:
return
try:
yield contact.ping()
except TimeoutError:
pass
except Exception as err:
log.warning("unexpected error: %s", err)
finally:
if contact in self._enqueued_contacts:
del self._enqueued_contacts[contact]
def _process(self):
# move contacts that are scheduled to join the queue
if self._pending_contacts:
now = self._node.clock.seconds()
for contact in [contact for contact, schedule in self._pending_contacts.items() if schedule <= now]:
del self._pending_contacts[contact]
self._enqueued_contacts.setdefault(contact, (defer.succeed(None), lambda: None))
# spread pings across 60 seconds to avoid flood and/or false negatives
step = 60.0/float(len(self._enqueued_contacts)) if self._enqueued_contacts else 0
for index, (contact, (call, _)) in enumerate(self._enqueued_contacts.items()):
if call.called and not contact.contact_is_good:
self._enqueued_contacts[contact] = self._node.reactor_callLater(index*step, self._ping, contact)
def start(self):
return self._node.safe_start_looping_call(self._process_lc, 60)
def stop(self):
map(None, (cancel() for _, (call, cancel) in self._enqueued_contacts.items() if not call.called))
return self._node.safe_stop_looping_call(self._process_lc)
class KademliaProtocol(protocol.DatagramProtocol):
""" Implements all low-level network-related functions of a Kademlia node """
msgSizeLimit = constants.udpDatagramMaxSize - 26
def __init__(self, node):
self._node = node
self._translator = msgformat.DefaultFormat()
self._sentMessages = {}
self._partialMessages = {}
self._partialMessagesProgress = {}
self._listening = defer.Deferred(None)
self._ping_queue = PingQueue(self._node)
self._protocolVersion = constants.protocolVersion
self.started_listening_time = 0
def _migrate_incoming_rpc_args(self, contact, method, *args):
if method == b'store' and contact.protocolVersion == 0:
if isinstance(args[1], dict):
blob_hash = args[0]
token = args[1].pop(b'token', None)
port = args[1].pop(b'port', -1)
originalPublisherID = args[1].pop(b'lbryid', None)
age = 0
return (blob_hash, token, port, originalPublisherID, age), {}
return args, {}
def _migrate_outgoing_rpc_args(self, contact, method, *args):
"""
This will reformat protocol version 0 arguments for the store function and will add the
protocol version keyword argument to calls to contacts who will accept it
"""
if contact.protocolVersion == 0:
if method == b'store':
blob_hash, token, port, originalPublisherID, age = args
args = (
blob_hash, {
b'token': token,
b'port': port,
b'lbryid': originalPublisherID
}, originalPublisherID, False
)
return args
return args
if args and isinstance(args[-1], dict):
args[-1][b'protocolVersion'] = self._protocolVersion
return args
return args + ({b'protocolVersion': self._protocolVersion},)
@defer.inlineCallbacks
def sendRPC(self, contact, method, args):
for _ in range(constants.rpcAttempts):
try:
response = yield self._sendRPC(contact, method, args)
return response
except TimeoutError:
if contact.contact_is_good:
log.debug("RETRY %s ON %s", method, contact)
continue
else:
raise
def _sendRPC(self, contact, method, args):
"""
Sends an RPC to the specified contact
@param contact: The contact (remote node) to send the RPC to
@type contact: kademlia.contacts.Contact
@param method: The name of remote method to invoke
@type method: str
@param args: A list of (non-keyword) arguments to pass to the remote
method, in the correct order
@type args: tuple
@return: This immediately returns a deferred object, which will return
the result of the RPC call, or raise the relevant exception
if the remote node raised one. If C{rawResponse} is set to
C{True}, however, it will always return the actual response
message (which may be a C{ResponseMessage} or an
C{ErrorMessage}).
@rtype: twisted.internet.defer.Deferred
"""
msg = msgtypes.RequestMessage(self._node.node_id, method, self._migrate_outgoing_rpc_args(contact, method,
*args))
msgPrimitive = self._translator.toPrimitive(msg)
encodedMsg = encoding.bencode(msgPrimitive)
if args:
log.debug("%s:%i SEND CALL %s(%s) TO %s:%i", self._node.externalIP, self._node.port, method,
hexlify(args[0]), contact.address, contact.port)
else:
log.debug("%s:%i SEND CALL %s TO %s:%i", self._node.externalIP, self._node.port, method,
contact.address, contact.port)
df = defer.Deferred()
def _remove_contact(failure): # remove the contact from the routing table and track the failure
contact.update_last_failed()
try:
if not contact.contact_is_good:
self._node.removeContact(contact)
except (ValueError, IndexError):
pass
return failure
def _update_contact(result): # refresh the contact in the routing table
contact.update_last_replied()
if method == b'findValue':
if b'token' in result:
contact.update_token(result[b'token'])
if b'protocolVersion' not in result:
contact.update_protocol_version(0)
else:
contact.update_protocol_version(result.pop(b'protocolVersion'))
d = self._node.addContact(contact)
d.addCallback(lambda _: result)
return d
df.addCallbacks(_update_contact, _remove_contact)
# Set the RPC timeout timer
timeoutCall, cancelTimeout = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, msg.id)
# Transmit the data
self._send(encodedMsg, msg.id, (contact.address, contact.port))
self._sentMessages[msg.id] = (contact, df, timeoutCall, cancelTimeout, method, args)
df.addErrback(cancelTimeout)
return df
def startProtocol(self):
log.info("DHT listening on UDP %i (ext port %i)", self._node.port, self._node.externalUDPPort)
if self._listening.called:
self._listening = defer.Deferred()
self._listening.callback(True)
self.started_listening_time = self._node.clock.seconds()
return self._ping_queue.start()
def datagramReceived(self, datagram, address):
""" Handles and parses incoming RPC messages (and responses)
@note: This is automatically called by Twisted when the protocol
receives a UDP datagram
"""
if chr(datagram[0]) == '\x00' and chr(datagram[25]) == '\x00':
totalPackets = (datagram[1] << 8) | datagram[2]
msgID = datagram[5:25]
seqNumber = (datagram[3] << 8) | datagram[4]
if msgID not in self._partialMessages:
self._partialMessages[msgID] = {}
self._partialMessages[msgID][seqNumber] = datagram[26:]
if len(self._partialMessages[msgID]) == totalPackets:
keys = self._partialMessages[msgID].keys()
keys.sort()
data = b''
for key in keys:
data += self._partialMessages[msgID][key]
datagram = data
del self._partialMessages[msgID]
else:
return
try:
msgPrimitive = encoding.bdecode(datagram)
message = self._translator.fromPrimitive(msgPrimitive)
except (encoding.DecodeError, ValueError) as err:
# We received some rubbish here
log.warning("Error decoding datagram %s from %s:%i - %s", hexlify(datagram),
address[0], address[1], err)
return
except (IndexError, KeyError):
log.warning("Couldn't decode dht datagram from %s", address)
return
if isinstance(message, msgtypes.RequestMessage):
# This is an RPC method request
remoteContact = self._node.contact_manager.make_contact(message.nodeID, address[0], address[1], self)
remoteContact.update_last_requested()
# only add a requesting contact to the routing table if it has replied to one of our requests
if remoteContact.contact_is_good is True:
df = self._node.addContact(remoteContact)
else:
df = defer.succeed(None)
df.addCallback(lambda _: self._handleRPC(remoteContact, message.id, message.request, message.args))
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
# will be added to our routing table if successful
if remoteContact.contact_is_good is None and remoteContact.lastReplied is None:
df.addCallback(lambda _: self._ping_queue.enqueue_maybe_ping(remoteContact))
elif isinstance(message, msgtypes.ErrorMessage):
# The RPC request raised a remote exception; raise it locally
if message.exceptionType in BUILTIN_EXCEPTIONS:
exception_type = BUILTIN_EXCEPTIONS[message.exceptionType]
else:
exception_type = UnknownRemoteException
remoteException = exception_type(message.response)
log.error("DHT RECV REMOTE EXCEPTION FROM %s:%i: %s", address[0],
address[1], remoteException)
if message.id in self._sentMessages:
# Cancel timeout timer for this RPC
remoteContact, df, timeoutCall, timeoutCanceller, method = self._sentMessages[message.id][0:5]
timeoutCanceller()
del self._sentMessages[message.id]
# reject replies coming from a different address than what we sent our request to
if (remoteContact.address, remoteContact.port) != address:
log.warning("Sent request to node %s at %s:%i, got reply from %s:%i",
remoteContact.log_id(), remoteContact.address,
remoteContact.port, address[0], address[1])
df.errback(TimeoutError(remoteContact.id))
return
# this error is returned by nodes that can be contacted but have an old
# and broken version of the ping command, if they return it the node can
# be contacted, so we'll treat it as a successful ping
old_ping_error = "ping() got an unexpected keyword argument '_rpcNodeContact'"
if isinstance(remoteException, TypeError) and \
remoteException.message == old_ping_error:
log.debug("old pong error")
df.callback('pong')
else:
df.errback(remoteException)
elif isinstance(message, msgtypes.ResponseMessage):
# Find the message that triggered this response
if message.id in self._sentMessages:
# Cancel timeout timer for this RPC
remoteContact, df, timeoutCall, timeoutCanceller, method = self._sentMessages[message.id][0:5]
timeoutCanceller()
del self._sentMessages[message.id]
log.debug("%s:%i RECV response to %s from %s:%i", self._node.externalIP, self._node.port,
method, remoteContact.address, remoteContact.port)
# When joining the network we made Contact objects for the seed nodes with node ids set to None
# Thus, the sent_to_id will also be None, and the contact objects need the ids to be manually set.
# These replies have be distinguished from those where the node id in the datagram does not match
# the node id of the node we sent a message to (these messages are treated as an error)
if remoteContact.id and remoteContact.id != message.nodeID: # sent_to_id will be None for bootstrap
log.debug("mismatch: (%s) %s:%i (%s vs %s)", method, remoteContact.address, remoteContact.port,
remoteContact.log_id(False), hexlify(message.nodeID))
df.errback(TimeoutError(remoteContact.id))
return
elif not remoteContact.id:
remoteContact.set_id(message.nodeID)
# We got a result from the RPC
df.callback(message.response)
else:
# If the original message isn't found, it must have timed out
# TODO: we should probably do something with this...
pass
def _send(self, data, rpcID, address):
""" Transmit the specified data over UDP, breaking it up into several
packets if necessary
If the data is spread over multiple UDP datagrams, the packets have the
following structure::
| | | | | |||||||||||| 0x00 |
|Transmission|Total number|Sequence number| RPC ID |Header end|
| type ID | of packets |of this packet | | indicator|
| (1 byte) | (2 bytes) | (2 bytes) |(20 bytes)| (1 byte) |
| | | | | |||||||||||| |
@note: The header used for breaking up large data segments will
possibly be moved out of the KademliaProtocol class in the
future, into something similar to a message translator/encoder
class (see C{kademlia.msgformat} and C{kademlia.encoding}).
"""
if len(data) > self.msgSizeLimit:
# We have to spread the data over multiple UDP datagrams,
# and provide sequencing information
#
# 1st byte is transmission type id, bytes 2 & 3 are the
# total number of packets in this transmission, bytes 4 &
# 5 are the sequence number for this specific packet
totalPackets = len(data) // self.msgSizeLimit
if len(data) % self.msgSizeLimit > 0:
totalPackets += 1
encTotalPackets = chr(totalPackets >> 8) + chr(totalPackets & 0xff)
seqNumber = 0
startPos = 0
while seqNumber < totalPackets:
packetData = data[startPos:startPos + self.msgSizeLimit]
encSeqNumber = chr(seqNumber >> 8) + chr(seqNumber & 0xff)
txData = f'\x00{encTotalPackets}{encSeqNumber}{rpcID}\x00{packetData}'
self._scheduleSendNext(txData, address)
startPos += self.msgSizeLimit
seqNumber += 1
else:
self._scheduleSendNext(data, address)
def _scheduleSendNext(self, txData, address):
"""Schedule the sending of the next UDP packet """
delayed_call, _ = self._node.reactor_callSoon(self._write, txData, address)
def _write(self, txData, address):
if self.transport:
try:
self.transport.write(txData, address)
except OSError as err:
if err.errno == errno.EWOULDBLOCK:
# i'm scared this may swallow important errors, but i get a million of these
# on Linux and it doesn't seem to affect anything -grin
log.warning("Can't send data to dht: EWOULDBLOCK")
elif err.errno == errno.ENETUNREACH:
# this should probably try to retransmit when the network connection is back
log.error("Network is unreachable")
else:
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
len(txData), address[0], address[1], err.message, err.errno)
raise err
else:
raise TransportNotConnected()
def _sendResponse(self, contact, rpcID, response):
""" Send a RPC response to the specified contact
"""
msg = msgtypes.ResponseMessage(rpcID, self._node.node_id, response)
msgPrimitive = self._translator.toPrimitive(msg)
encodedMsg = encoding.bencode(msgPrimitive)
self._send(encodedMsg, rpcID, (contact.address, contact.port))
def _sendError(self, contact, rpcID, exceptionType, exceptionMessage):
""" Send an RPC error message to the specified contact
"""
exceptionMessage = exceptionMessage.encode()
msg = msgtypes.ErrorMessage(rpcID, self._node.node_id, exceptionType, exceptionMessage)
msgPrimitive = self._translator.toPrimitive(msg)
encodedMsg = encoding.bencode(msgPrimitive)
self._send(encodedMsg, rpcID, (contact.address, contact.port))
def _handleRPC(self, senderContact, rpcID, method, args):
""" Executes a local function in response to an RPC request """
# Set up the deferred callchain
def handleError(f):
self._sendError(senderContact, rpcID, f.type, f.getErrorMessage())
def handleResult(result):
self._sendResponse(senderContact, rpcID, result)
df = defer.Deferred()
df.addCallback(handleResult)
df.addErrback(handleError)
# Execute the RPC
func = getattr(self._node, method.decode(), None)
if callable(func) and hasattr(func, "rpcmethod"):
# Call the exposed Node method and return the result to the deferred callback chain
# if args:
# log.debug("%s:%i RECV CALL %s(%s) %s:%i", self._node.externalIP, self._node.port, method,
# args[0].encode('hex'), senderContact.address, senderContact.port)
# else:
log.debug("%s:%i RECV CALL %s %s:%i", self._node.externalIP, self._node.port, method,
senderContact.address, senderContact.port)
if args and isinstance(args[-1], dict) and b'protocolVersion' in args[-1]: # args don't need reformatting
senderContact.update_protocol_version(int(args[-1].pop(b'protocolVersion')))
a, kw = tuple(args[:-1]), args[-1]
else:
senderContact.update_protocol_version(0)
a, kw = self._migrate_incoming_rpc_args(senderContact, method, *args)
try:
if method != b'ping':
result = func(senderContact, *a)
else:
result = func()
except Exception as e:
log.error("error handling request for %s:%i %s", senderContact.address, senderContact.port, method)
df.errback(e)
else:
df.callback(result)
else:
# No such exposed method
df.errback(AttributeError('Invalid method: %s' % method))
return df
def _msgTimeout(self, messageID):
""" Called when an RPC request message times out """
# Find the message that timed out
if messageID not in self._sentMessages:
# This should never be reached
log.error("deferred timed out, but is not present in sent messages list!")
return
remoteContact, df, timeout_call, timeout_canceller, method, args = self._sentMessages[messageID]
if messageID in self._partialMessages:
# We are still receiving this message
self._msgTimeoutInProgress(messageID, timeout_canceller, remoteContact, df, method, args)
return
del self._sentMessages[messageID]
# The message's destination node is now considered to be dead;
# raise an (asynchronous) TimeoutError exception and update the host node
df.errback(TimeoutError(remoteContact.id))
def _msgTimeoutInProgress(self, messageID, timeoutCanceller, remoteContact, df, method, args):
# See if any progress has been made; if not, kill the message
if self._hasProgressBeenMade(messageID):
# Reset the RPC timeout timer
timeoutCanceller()
timeoutCall, cancelTimeout = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, messageID)
self._sentMessages[messageID] = (remoteContact, df, timeoutCall, cancelTimeout, method, args)
else:
# No progress has been made
if messageID in self._partialMessagesProgress:
del self._partialMessagesProgress[messageID]
if messageID in self._partialMessages:
del self._partialMessages[messageID]
df.errback(TimeoutError(remoteContact.id))
def _hasProgressBeenMade(self, messageID):
return (
messageID in self._partialMessagesProgress and
(
len(self._partialMessagesProgress[messageID]) !=
len(self._partialMessages[messageID])
)
)
def stopProtocol(self):
""" Called when the transport is disconnected.
Will only be called once, after all ports are disconnected.
"""
log.info('Stopping DHT')
self._ping_queue.stop()
self._node.call_later_manager.stop()
log.info('DHT stopped')
-70
View File
@@ -1,70 +0,0 @@
import asyncio
import typing
from lbrynet.dht import constants
if typing.TYPE_CHECKING:
from lbrynet.dht.peer import KademliaPeer, PeerManager
class DictDataStore:
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager'):
# Dictionary format:
# { <key>: [(<contact>, <age>), ...] }
self._data_store: typing.Dict[bytes, typing.List[typing.Tuple['KademliaPeer', float]]] = {}
self.loop = loop
self._peer_manager = peer_manager
self.completed_blobs: typing.Set[str] = set()
def removed_expired_peers(self):
now = self.loop.time()
keys = list(self._data_store.keys())
for key in keys:
to_remove = []
for (peer, ts) in self._data_store[key]:
if ts + constants.data_expiration < now or self._peer_manager.peer_is_good(peer) is False:
to_remove.append((peer, ts))
for item in to_remove:
self._data_store[key].remove(item)
if not self._data_store[key]:
del self._data_store[key]
def filter_bad_and_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
"""
Returns only non-expired and unknown/good peers
"""
for peer in self.filter_expired_peers(key):
if self._peer_manager.peer_is_good(peer) is not False:
yield peer
def filter_expired_peers(self, key: bytes) -> typing.Iterator['KademliaPeer']:
"""
Returns only non-expired peers
"""
now = self.loop.time()
for (peer, ts) in self._data_store.get(key, []):
if ts + constants.data_expiration > now:
yield peer
def has_peers_for_blob(self, key: bytes) -> bool:
return key in self._data_store
def add_peer_to_blob(self, contact: 'KademliaPeer', key: bytes) -> None:
now = self.loop.time()
if key in self._data_store:
current = list(filter(lambda x: x[0] == contact, self._data_store[key]))
if len(current):
self._data_store[key][self._data_store[key].index(current[0])] = contact, now
else:
self._data_store[key].append((contact, now))
else:
self._data_store[key] = [(contact, now)]
def get_peers_for_blob(self, key: bytes) -> typing.List['KademliaPeer']:
return list(self.filter_bad_and_expired_peers(key))
def get_storing_contacts(self) -> typing.List['KademliaPeer']:
peers = set()
for key, stored in self._data_store.items():
peers.update(set(map(lambda tup: tup[0], stored)))
return list(peers)
-347
View File
@@ -1,347 +0,0 @@
import asyncio
from binascii import hexlify
from itertools import chain
import typing
import logging
from lbrynet.dht import constants
from lbrynet.dht.error import RemoteException, TransportNotConnected
from lbrynet.dht.protocol.distance import Distance
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from lbrynet.dht.protocol.routing_table import TreeRoutingTable
from lbrynet.dht.protocol.protocol import KademliaProtocol
from lbrynet.dht.peer import PeerManager, KademliaPeer
log = logging.getLogger(__name__)
class FindResponse:
@property
def found(self) -> bool:
raise NotImplementedError()
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
raise NotImplementedError()
class FindNodeResponse(FindResponse):
def __init__(self, key: bytes, close_triples: typing.List[typing.Tuple[bytes, str, int]]):
self.key = key
self.close_triples = close_triples
@property
def found(self) -> bool:
return self.key in [triple[0] for triple in self.close_triples]
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
return self.close_triples
class FindValueResponse(FindResponse):
def __init__(self, key: bytes, result_dict: typing.Dict):
self.key = key
self.token = result_dict[b'token']
self.close_triples: typing.List[typing.Tuple[bytes, bytes, int]] = result_dict.get(b'contacts', [])
self.found_compact_addresses = result_dict.get(key, [])
@property
def found(self) -> bool:
return len(self.found_compact_addresses) > 0
def get_close_triples(self) -> typing.List[typing.Tuple[bytes, str, int]]:
return [(node_id, address.decode(), port) for node_id, address, port in self.close_triples]
def get_shortlist(routing_table: 'TreeRoutingTable', key: bytes,
shortlist: typing.Optional[typing.List['KademliaPeer']]) -> typing.List['KademliaPeer']:
"""
If not provided, initialize the shortlist of peers to probe to the (up to) k closest peers in the routing table
:param routing_table: a TreeRoutingTable
:param key: a 48 byte hash
:param shortlist: optional manually provided shortlist, this is done during bootstrapping when there are no
peers in the routing table. During bootstrap the shortlist is set to be the seed nodes.
"""
if len(key) != constants.hash_length:
raise ValueError("invalid key length: %i" % len(key))
return shortlist or routing_table.find_close_peers(key)
class IterativeFinder:
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager',
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
if len(key) != constants.hash_length:
raise ValueError("invalid key length: %i" % len(key))
self.loop = loop
self.peer_manager = peer_manager
self.routing_table = routing_table
self.protocol = protocol
self.key = key
self.bottom_out_limit = bottom_out_limit
self.max_results = max_results
self.exclude = exclude or []
self.active: typing.Set['KademliaPeer'] = set()
self.contacted: typing.Set['KademliaPeer'] = set()
self.distance = Distance(key)
self.closest_peer: typing.Optional['KademliaPeer'] = None
self.prev_closest_peer: typing.Optional['KademliaPeer'] = None
self.iteration_queue = asyncio.Queue(loop=self.loop)
self.running_probes: typing.Set[asyncio.Task] = set()
self.iteration_count = 0
self.bottom_out_count = 0
self.running = False
self.tasks: typing.List[asyncio.Task] = []
self.delayed_calls: typing.List[asyncio.Handle] = []
for peer in get_shortlist(routing_table, key, shortlist):
if peer.node_id:
self._add_active(peer)
else:
# seed nodes
self._schedule_probe(peer)
async def send_probe(self, peer: 'KademliaPeer') -> FindResponse:
"""
Send the rpc request to the peer and return an object with the FindResponse interface
"""
raise NotImplementedError()
def search_exhausted(self):
"""
This method ends the iterator due no more peers to contact.
Override to provide last time results.
"""
self.iteration_queue.put_nowait(None)
def check_result_ready(self, response: FindResponse):
"""
Called after adding peers from an rpc result to the shortlist.
This method is responsible for putting a result for the generator into the Queue
"""
raise NotImplementedError()
def get_initial_result(self) -> typing.List['KademliaPeer']:
"""
Get an initial or cached result to be put into the Queue. Used for findValue requests where the blob
has peers in the local data store of blobs announced to us
"""
return []
def _is_closer(self, peer: 'KademliaPeer') -> bool:
return not self.closest_peer or self.distance.is_closer(peer.node_id, self.closest_peer.node_id)
def _add_active(self, peer):
if peer not in self.active and peer.node_id and peer.node_id != self.protocol.node_id:
self.active.add(peer)
if self._is_closer(peer):
self.prev_closest_peer = self.closest_peer
self.closest_peer = peer
async def _handle_probe_result(self, peer: 'KademliaPeer', response: FindResponse):
self._add_active(peer)
for contact_triple in response.get_close_triples():
node_id, address, udp_port = contact_triple
self._add_active(self.peer_manager.get_kademlia_peer(node_id, address, udp_port))
self.check_result_ready(response)
async def _send_probe(self, peer: 'KademliaPeer'):
try:
response = await self.send_probe(peer)
except asyncio.TimeoutError:
self.active.discard(peer)
return
except ValueError as err:
log.warning(str(err))
self.active.discard(peer)
return
except TransportNotConnected:
return self.aclose()
except RemoteException:
return
return await self._handle_probe_result(peer, response)
async def _search_round(self):
"""
Send up to constants.alpha (5) probes to closest active peers
"""
added = 0
to_probe = list(self.active - self.contacted)
to_probe.sort(key=lambda peer: self.distance(self.key))
for peer in to_probe:
if added >= constants.alpha:
break
origin_address = (peer.address, peer.udp_port)
if origin_address in self.exclude:
continue
if peer.node_id == self.protocol.node_id:
continue
if origin_address == (self.protocol.external_ip, self.protocol.udp_port):
continue
self._schedule_probe(peer)
added += 1
log.debug("running %d probes", len(self.running_probes))
if not added and not self.running_probes:
log.debug("search for %s exhausted", hexlify(self.key)[:8])
self.search_exhausted()
def _schedule_probe(self, peer: 'KademliaPeer'):
self.contacted.add(peer)
t = self.loop.create_task(self._send_probe(peer))
def callback(_):
self.running_probes.difference_update({
probe for probe in self.running_probes if probe.done() or probe == t
})
if not self.running_probes:
self.tasks.append(self.loop.create_task(self._search_task(0.0)))
t.add_done_callback(callback)
self.running_probes.add(t)
async def _search_task(self, delay: typing.Optional[float] = constants.iterative_lookup_delay):
try:
if self.running:
await self._search_round()
if self.running:
self.delayed_calls.append(self.loop.call_later(delay, self._search))
except (asyncio.CancelledError, StopAsyncIteration, TransportNotConnected):
if self.running:
self.loop.call_soon(self.aclose)
def _search(self):
self.tasks.append(self.loop.create_task(self._search_task()))
def __aiter__(self):
if self.running:
raise Exception("already running")
self.running = True
self._search()
return self
async def __anext__(self) -> typing.List['KademliaPeer']:
try:
if self.iteration_count == 0:
result = self.get_initial_result() or await self.iteration_queue.get()
else:
result = await self.iteration_queue.get()
if not result:
raise StopAsyncIteration
self.iteration_count += 1
return result
except (asyncio.CancelledError, StopAsyncIteration):
self.loop.call_soon(self.aclose)
raise
def aclose(self):
self.running = False
self.iteration_queue.put_nowait(None)
for task in chain(self.tasks, self.running_probes, self.delayed_calls):
task.cancel()
self.tasks.clear()
self.running_probes.clear()
self.delayed_calls.clear()
class IterativeNodeFinder(IterativeFinder):
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager',
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
shortlist)
self.yielded_peers: typing.Set['KademliaPeer'] = set()
async def send_probe(self, peer: 'KademliaPeer') -> FindNodeResponse:
log.debug("probing %s:%d %s", peer.address, peer.udp_port, hexlify(peer.node_id)[:8] if peer.node_id else '')
response = await self.protocol.get_rpc_peer(peer).find_node(self.key)
return FindNodeResponse(self.key, response)
def search_exhausted(self):
self.put_result(self.active, finish=True)
def put_result(self, from_iter: typing.Iterable['KademliaPeer'], finish=False):
not_yet_yielded = [
peer for peer in from_iter
if peer not in self.yielded_peers
and peer.node_id != self.protocol.node_id
and self.peer_manager.peer_is_good(peer) is not False
]
not_yet_yielded.sort(key=lambda peer: self.distance(peer.node_id))
to_yield = not_yet_yielded[:min(constants.k, len(not_yet_yielded))]
if to_yield:
self.yielded_peers.update(to_yield)
self.iteration_queue.put_nowait(to_yield)
if finish:
self.iteration_queue.put_nowait(None)
def check_result_ready(self, response: FindNodeResponse):
found = response.found and self.key != self.protocol.node_id
if found:
log.debug("found")
return self.put_result(self.active, finish=True)
if self.prev_closest_peer and self.closest_peer and not self._is_closer(self.prev_closest_peer):
# log.info("improving, %i %i %i %i %i", len(self.shortlist), len(self.active), len(self.contacted),
# self.bottom_out_count, self.iteration_count)
self.bottom_out_count = 0
elif self.prev_closest_peer and self.closest_peer:
self.bottom_out_count += 1
log.info("bottom out %i %i %i", len(self.active), len(self.contacted), self.bottom_out_count)
if self.bottom_out_count >= self.bottom_out_limit or self.iteration_count >= self.bottom_out_limit:
log.info("limit hit")
self.put_result(self.active, True)
class IterativeValueFinder(IterativeFinder):
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager',
routing_table: 'TreeRoutingTable', protocol: 'KademliaProtocol', key: bytes,
bottom_out_limit: typing.Optional[int] = 2, max_results: typing.Optional[int] = constants.k,
exclude: typing.Optional[typing.List[typing.Tuple[str, int]]] = None,
shortlist: typing.Optional[typing.List['KademliaPeer']] = None):
super().__init__(loop, peer_manager, routing_table, protocol, key, bottom_out_limit, max_results, exclude,
shortlist)
self.blob_peers: typing.Set['KademliaPeer'] = set()
async def send_probe(self, peer: 'KademliaPeer') -> FindValueResponse:
response = await self.protocol.get_rpc_peer(peer).find_value(self.key)
return FindValueResponse(self.key, response)
def check_result_ready(self, response: FindValueResponse):
if response.found:
blob_peers = [self.peer_manager.decode_tcp_peer_from_compact_address(compact_addr)
for compact_addr in response.found_compact_addresses]
to_yield = []
self.bottom_out_count = 0
for blob_peer in blob_peers:
if blob_peer not in self.blob_peers:
self.blob_peers.add(blob_peer)
to_yield.append(blob_peer)
if to_yield:
# log.info("found %i new peers for blob", len(to_yield))
self.iteration_queue.put_nowait(to_yield)
# if self.max_results and len(self.blob_peers) >= self.max_results:
# log.info("enough blob peers found")
# if not self.finished.is_set():
# self.finished.set()
elif self.prev_closest_peer and self.closest_peer:
self.bottom_out_count += 1
if self.bottom_out_count >= self.bottom_out_limit:
log.info("blob peer search bottomed out")
self.iteration_queue.put_nowait(None)
def get_initial_result(self) -> typing.List['KademliaPeer']:
if self.protocol.data_store.has_peers_for_blob(self.key):
return self.protocol.data_store.get_peers_for_blob(self.key)
return []
-673
View File
@@ -1,673 +0,0 @@
import logging
import socket
import functools
import hashlib
import asyncio
import typing
import binascii
from asyncio.protocols import DatagramProtocol
from asyncio.transports import DatagramTransport
from lbrynet.dht import constants
from lbrynet.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram
from lbrynet.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE
from lbrynet.dht.error import RemoteException, TransportNotConnected
from lbrynet.dht.protocol.routing_table import TreeRoutingTable
from lbrynet.dht.protocol.data_store import DictDataStore
if typing.TYPE_CHECKING:
from lbrynet.dht.peer import PeerManager, KademliaPeer
log = logging.getLogger(__name__)
old_protocol_errors = {
"findNode() takes exactly 2 arguments (5 given)": "0.19.1",
"findValue() takes exactly 2 arguments (5 given)": "0.19.1"
}
class KademliaRPC:
def __init__(self, protocol: 'KademliaProtocol', loop: asyncio.BaseEventLoop, peer_port: int = 3333):
self.protocol = protocol
self.loop = loop
self.peer_port = peer_port
self.old_token_secret: bytes = None
self.token_secret = constants.generate_id()
def compact_address(self):
compact_ip = functools.reduce(lambda buff, x: buff + bytearray([int(x)]),
self.protocol.external_ip.split('.'), bytearray())
compact_port = self.peer_port.to_bytes(2, 'big')
return compact_ip + compact_port + self.protocol.node_id
@staticmethod
def ping():
return b'pong'
def store(self, rpc_contact: 'KademliaPeer', blob_hash: bytes, token: bytes, port: int) -> bytes:
if len(blob_hash) != constants.hash_bits // 8:
raise ValueError(f"invalid length of blob hash: {len(blob_hash)}")
if not 0 < port < 65535:
raise ValueError(f"invalid tcp port: {port}")
rpc_contact.update_tcp_port(port)
if not self.verify_token(token, rpc_contact.compact_ip()):
if self.loop.time() - self.protocol.started_listening_time < constants.token_secret_refresh_interval:
pass
else:
raise ValueError("Invalid token")
self.protocol.data_store.add_peer_to_blob(
rpc_contact, blob_hash
)
return b'OK'
def find_node(self, rpc_contact: 'KademliaPeer', key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
if len(key) != constants.hash_length:
raise ValueError("invalid contact node_id length: %i" % len(key))
contacts = self.protocol.routing_table.find_close_peers(key, sender_node_id=rpc_contact.node_id)
contact_triples = []
for contact in contacts:
contact_triples.append((contact.node_id, contact.address, contact.udp_port))
return contact_triples
def find_value(self, rpc_contact: 'KademliaPeer', key: bytes):
if len(key) != constants.hash_length:
raise ValueError("invalid blob_exchange hash length: %i" % len(key))
response = {
b'token': self.make_token(rpc_contact.compact_ip()),
b'contacts': self.find_node(rpc_contact, key)
}
if self.protocol.protocol_version:
response[b'protocolVersion'] = self.protocol.protocol_version
# get peers we have stored for this blob_exchange
peers = [
peer.compact_address_tcp()
for peer in self.protocol.data_store.get_peers_for_blob(key)
if not rpc_contact.tcp_port or peer.compact_address_tcp() != rpc_contact.compact_address_tcp()
]
# if we don't have k storing peers to return and we have this hash locally, include our contact information
if len(peers) < constants.k and binascii.hexlify(key).decode() in self.protocol.data_store.completed_blobs:
peers.append(self.compact_address())
if peers:
response[key] = peers
return response
def refresh_token(self): # TODO: this needs to be called periodically
self.old_token_secret = self.token_secret
self.token_secret = constants.generate_id()
def make_token(self, compact_ip):
h = hashlib.new('sha384')
h.update(self.token_secret + compact_ip)
return h.digest()
def verify_token(self, token, compact_ip):
h = hashlib.new('sha384')
h.update(self.token_secret + compact_ip)
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
h = hashlib.new('sha384')
h.update(self.old_token_secret + compact_ip)
if not token == h.digest():
return False
return True
class RemoteKademliaRPC:
"""
Encapsulates RPC calls to remote Peers
"""
def __init__(self, loop: asyncio.BaseEventLoop, peer_tracker: 'PeerManager', protocol: 'KademliaProtocol',
peer: 'KademliaPeer'):
self.loop = loop
self.peer_tracker = peer_tracker
self.protocol = protocol
self.peer = peer
async def ping(self) -> bytes:
"""
:return: b'pong'
"""
response = await self.protocol.send_request(
self.peer, RequestDatagram.make_ping(self.protocol.node_id)
)
return response.response
async def store(self, blob_hash: bytes) -> bytes:
"""
:param blob_hash: blob hash as bytes
:return: b'OK'
"""
if len(blob_hash) != constants.hash_bits // 8:
raise ValueError(f"invalid length of blob hash: {len(blob_hash)}")
if not self.protocol.peer_port or not 0 < self.protocol.peer_port < 65535:
raise ValueError(f"invalid tcp port: {self.protocol.peer_port}")
token = self.peer_tracker.get_node_token(self.peer.node_id)
if not token:
find_value_resp = await self.find_value(blob_hash)
token = find_value_resp[b'token']
response = await self.protocol.send_request(
self.peer, RequestDatagram.make_store(self.protocol.node_id, blob_hash, token, self.protocol.peer_port)
)
return response.response
async def find_node(self, key: bytes) -> typing.List[typing.Tuple[bytes, str, int]]:
"""
:return: [(node_id, address, udp_port), ...]
"""
if len(key) != constants.hash_bits // 8:
raise ValueError(f"invalid length of find node key: {len(key)}")
response = await self.protocol.send_request(
self.peer, RequestDatagram.make_find_node(self.protocol.node_id, key)
)
return [(node_id, address.decode(), udp_port) for node_id, address, udp_port in response.response]
async def find_value(self, key: bytes) -> typing.Union[typing.Dict]:
"""
:return: {
b'token': <token bytes>,
b'contacts': [(node_id, address, udp_port), ...]
<key bytes>: [<blob_peer_compact_address, ...]
}
"""
if len(key) != constants.hash_bits // 8:
raise ValueError(f"invalid length of find value key: {len(key)}")
response = await self.protocol.send_request(
self.peer, RequestDatagram.make_find_value(self.protocol.node_id, key)
)
self.peer_tracker.update_token(self.peer.node_id, response.response[b'token'])
return response.response
class PingQueue:
def __init__(self, loop: asyncio.BaseEventLoop, protocol: 'KademliaProtocol'):
self._loop = loop
self._protocol = protocol
self._pending_contacts: typing.Dict['KademliaPeer', float] = {}
self._process_task: asyncio.Task = None
self._running = False
self._running_pings: typing.Set[asyncio.Task] = set()
self._default_delay = constants.maybe_ping_delay
@property
def running(self):
return self._running
def enqueue_maybe_ping(self, *peers: 'KademliaPeer', delay: typing.Optional[float] = None):
delay = delay if delay is not None else self._default_delay
now = self._loop.time()
for peer in peers:
if peer not in self._pending_contacts or now + delay < self._pending_contacts[peer]:
self._pending_contacts[peer] = delay + now
def maybe_ping(self, peer: 'KademliaPeer'):
async def ping_task():
try:
if self._protocol.peer_manager.peer_is_good(peer):
if peer not in self._protocol.routing_table.get_peers():
self._protocol.add_peer(peer)
return
await self._protocol.get_rpc_peer(peer).ping()
except asyncio.TimeoutError:
pass
task = self._loop.create_task(ping_task())
task.add_done_callback(lambda _: None if task not in self._running_pings else self._running_pings.remove(task))
self._running_pings.add(task)
async def _process(self): # send up to 1 ping per second
while True:
enqueued = list(self._pending_contacts.keys())
now = self._loop.time()
for peer in enqueued:
if self._pending_contacts[peer] <= now:
del self._pending_contacts[peer]
self.maybe_ping(peer)
break
await asyncio.sleep(1, loop=self._loop)
def start(self):
assert not self._running
self._running = True
if not self._process_task:
self._process_task = self._loop.create_task(self._process())
def stop(self):
assert self._running
self._running = False
if self._process_task:
self._process_task.cancel()
self._process_task = None
while self._running_pings:
self._running_pings.pop().cancel()
class KademliaProtocol(DatagramProtocol):
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', node_id: bytes, external_ip: str,
udp_port: int, peer_port: int, rpc_timeout: float = constants.rpc_timeout,
split_buckets_under_index: int = constants.split_buckets_under_index):
self.peer_manager = peer_manager
self.loop = loop
self.node_id = node_id
self.external_ip = external_ip
self.udp_port = udp_port
self.peer_port = peer_port
self.is_seed_node = False
self.partial_messages: typing.Dict[bytes, typing.Dict[bytes, bytes]] = {}
self.sent_messages: typing.Dict[bytes, typing.Tuple['KademliaPeer', asyncio.Future, RequestDatagram]] = {}
self.protocol_version = constants.protocol_version
self.started_listening_time = 0
self.transport: DatagramTransport = None
self.old_token_secret = constants.generate_id()
self.token_secret = constants.generate_id()
self.routing_table = TreeRoutingTable(self.loop, self.peer_manager, self.node_id, split_buckets_under_index)
self.data_store = DictDataStore(self.loop, self.peer_manager)
self.ping_queue = PingQueue(self.loop, self)
self.node_rpc = KademliaRPC(self, self.loop, self.peer_port)
self.rpc_timeout = rpc_timeout
self._split_lock = asyncio.Lock(loop=self.loop)
self._to_remove: typing.Set['KademliaPeer'] = set()
self._to_add: typing.Set['KademliaPeer'] = set()
self._wakeup_routing_task = asyncio.Event(loop=self.loop)
self.maintaing_routing_task: typing.Optional[asyncio.Task] = None
@functools.lru_cache(128)
def get_rpc_peer(self, peer: 'KademliaPeer') -> RemoteKademliaRPC:
return RemoteKademliaRPC(self.loop, self.peer_manager, self, peer)
def start(self):
self.maintaing_routing_task = self.loop.create_task(self.routing_table_task())
def stop(self):
if self.maintaing_routing_task:
self.maintaing_routing_task.cancel()
if self.transport:
self.disconnect()
def disconnect(self):
self.transport.close()
def connection_made(self, transport: DatagramTransport):
self.transport = transport
def connection_lost(self, exc):
self.stop()
@staticmethod
def _migrate_incoming_rpc_args(peer: 'KademliaPeer', method: bytes, *args) -> typing.Tuple[typing.Tuple,
typing.Dict]:
if method == b'store' and peer.protocol_version == 0:
if isinstance(args[1], dict):
blob_hash = args[0]
token = args[1].pop(b'token', None)
port = args[1].pop(b'port', -1)
original_publisher_id = args[1].pop(b'lbryid', None)
age = 0
return (blob_hash, token, port, original_publisher_id, age), {}
return args, {}
async def _add_peer(self, peer: 'KademliaPeer'):
for p in self.routing_table.get_peers():
if (p.address, p.udp_port) == (peer.address, peer.udp_port) and p.node_id != peer.node_id:
self.routing_table.remove_peer(p)
self.routing_table.join_buckets()
bucket_index = self.routing_table.kbucket_index(peer.node_id)
if self.routing_table.buckets[bucket_index].add_peer(peer):
return True
# The bucket is full; see if it can be split (by checking if its range includes the host node's node_id)
if self.routing_table.should_split(bucket_index, peer.node_id):
self.routing_table.split_bucket(bucket_index)
# Retry the insertion attempt
result = await self._add_peer(peer)
self.routing_table.join_buckets()
return result
else:
# We can't split the k-bucket
#
# The 13 page kademlia paper specifies that the least recently contacted node in the bucket
# shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
# the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
#
# A reasonable extension to this is BEP 0005, which extends the above:
#
# Not all nodes that we learn about are equal. Some are "good" and some are not.
# Many nodes using the DHT are able to send queries and receive responses,
# but are not able to respond to queries from other nodes. It is important that
# each node's routing table must contain only known good nodes. A good node is
# a node has responded to one of our queries within the last 15 minutes. A node
# is also good if it has ever responded to one of our queries and has sent us a
# query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
# questionable. Nodes become bad when they fail to respond to multiple queries
# in a row. Nodes that we know are good are given priority over nodes with unknown status.
#
# When there are bad or questionable nodes in the bucket, the least recent is selected for
# potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
# contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
# is ignored if the pinged node replies.
not_good_contacts = self.routing_table.buckets[bucket_index].get_bad_or_unknown_peers()
not_recently_replied = []
for p in not_good_contacts:
last_replied = self.peer_manager.get_last_replied(p.address, p.udp_port)
if not last_replied or last_replied + 60 < self.loop.time():
not_recently_replied.append(p)
if not_recently_replied:
to_replace = not_recently_replied[0]
else:
to_replace = self.routing_table.buckets[bucket_index].peers[0]
last_replied = self.peer_manager.get_last_replied(to_replace.address, to_replace.udp_port)
if last_replied and last_replied + 60 > self.loop.time():
return False
log.debug("pinging %s:%s", to_replace.address, to_replace.udp_port)
try:
to_replace_rpc = self.get_rpc_peer(to_replace)
await to_replace_rpc.ping()
return False
except asyncio.TimeoutError:
log.debug("Replacing dead contact in bucket %i: %s:%i with %s:%i ", bucket_index,
to_replace.address, to_replace.udp_port, peer.address, peer.udp_port)
if to_replace in self.routing_table.buckets[bucket_index]:
self.routing_table.buckets[bucket_index].remove_peer(to_replace)
return await self._add_peer(peer)
def add_peer(self, peer: 'KademliaPeer'):
if peer.node_id == self.node_id:
return False
self._to_add.add(peer)
self._wakeup_routing_task.set()
def remove_peer(self, peer: 'KademliaPeer'):
self._to_remove.add(peer)
self._wakeup_routing_task.set()
async def routing_table_task(self):
while True:
while self._to_remove:
async with self._split_lock:
peer = self._to_remove.pop()
self.routing_table.remove_peer(peer)
self.routing_table.join_buckets()
while self._to_add:
async with self._split_lock:
await self._add_peer(self._to_add.pop())
await asyncio.gather(self._wakeup_routing_task.wait(), asyncio.sleep(.1, loop=self.loop), loop=self.loop)
self._wakeup_routing_task.clear()
def _handle_rpc(self, sender_contact: 'KademliaPeer', message: RequestDatagram):
assert sender_contact.node_id != self.node_id, (binascii.hexlify(sender_contact.node_id)[:8].decode(),
binascii.hexlify(self.node_id)[:8].decode())
method = message.method
if method not in [b'ping', b'store', b'findNode', b'findValue']:
raise AttributeError('Invalid method: %s' % message.method.decode())
if message.args and isinstance(message.args[-1], dict) and b'protocolVersion' in message.args[-1]:
# args don't need reformatting
a, kw = tuple(message.args[:-1]), message.args[-1]
else:
a, kw = self._migrate_incoming_rpc_args(sender_contact, message.method, *message.args)
log.debug("%s:%i RECV CALL %s %s:%i", self.external_ip, self.udp_port, message.method.decode(),
sender_contact.address, sender_contact.udp_port)
if method == b'ping':
result = self.node_rpc.ping()
elif method == b'store':
blob_hash, token, port, original_publisher_id, age = a
result = self.node_rpc.store(sender_contact, blob_hash, token, port)
elif method == b'findNode':
key, = a
result = self.node_rpc.find_node(sender_contact, key)
else:
assert method == b'findValue'
key, = a
result = self.node_rpc.find_value(sender_contact, key)
self.send_response(
sender_contact, ResponseDatagram(RESPONSE_TYPE, message.rpc_id, self.node_id, result),
)
def handle_request_datagram(self, address: typing.Tuple[str, int], request_datagram: RequestDatagram):
# This is an RPC method request
self.peer_manager.report_last_requested(address[0], address[1])
try:
peer = self.routing_table.get_peer(request_datagram.node_id)
except IndexError:
peer = self.peer_manager.get_kademlia_peer(request_datagram.node_id, address[0], address[1])
try:
self._handle_rpc(peer, request_datagram)
# if the contact is not known to be bad (yet) and we haven't yet queried it, send it a ping so that it
# will be added to our routing table if successful
is_good = self.peer_manager.peer_is_good(peer)
if is_good is None:
self.ping_queue.enqueue_maybe_ping(peer)
# only add a requesting contact to the routing table if it has replied to one of our requests
elif is_good is True:
self.add_peer(peer)
except ValueError as err:
log.debug("error raised handling %s request from %s:%i - %s(%s)",
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
str(err))
self.send_error(
peer,
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
str(err).encode())
)
except Exception as err:
log.warning("error raised handling %s request from %s:%i - %s(%s)",
request_datagram.method, peer.address, peer.udp_port, str(type(err)),
str(err))
self.send_error(
peer,
ErrorDatagram(ERROR_TYPE, request_datagram.rpc_id, self.node_id, str(type(err)).encode(),
str(err).encode())
)
def handle_response_datagram(self, address: typing.Tuple[str, int], response_datagram: ResponseDatagram):
# Find the message that triggered this response
if response_datagram.rpc_id in self.sent_messages:
peer, df, request = self.sent_messages[response_datagram.rpc_id]
if peer.address != address[0]:
df.set_exception(RemoteException(
f"response from {address[0]}, expected {peer.address}")
)
return
# We got a result from the RPC
if peer.node_id == self.node_id:
df.set_exception(RemoteException("node has our node id"))
return
elif response_datagram.node_id == self.node_id:
df.set_exception(RemoteException("incoming message is from our node id"))
return
peer.set_id(response_datagram.node_id)
peer.update_udp_port(address[1])
self.peer_manager.report_last_replied(address[0], address[1])
self.peer_manager.update_contact_triple(peer.node_id, address[0], address[1])
if not df.cancelled():
df.set_result(response_datagram)
self.add_peer(peer)
else:
log.warning("%s:%i replied, but after we cancelled the request attempt",
peer.address, peer.udp_port)
else:
# If the original message isn't found, it must have timed out
# TODO: we should probably do something with this...
pass
def handle_error_datagram(self, address, error_datagram: ErrorDatagram):
# The RPC request raised a remote exception; raise it locally
remote_exception = RemoteException(f"{error_datagram.exception_type}({error_datagram.response})")
if error_datagram.rpc_id in self.sent_messages:
peer, df, request = self.sent_messages.pop(error_datagram.rpc_id)
if (peer.address, peer.udp_port) != address:
df.set_exception(RemoteException(
f"response from {address[0]}:{address[1]}, "
f"expected {peer.address}:{peer.udp_port}")
)
return
error_msg = f"" \
f"Error sending '{request.method}' to {peer.address}:{peer.udp_port}\n" \
f"Args: {request.args}\n" \
f"Raised: {str(remote_exception)}"
if 'Invalid token' in error_msg:
log.debug(error_msg)
elif error_datagram.response not in old_protocol_errors:
log.warning(error_msg)
else:
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
peer.address, peer.udp_port, old_protocol_errors[error_datagram.response])
df.set_exception(remote_exception)
return
else:
if error_datagram.response not in old_protocol_errors:
msg = f"Received error from {address[0]}:{address[1]}, but it isn't in response to a " \
f"pending request: {str(remote_exception)}"
log.warning(msg)
else:
log.warning("known dht protocol backwards compatibility error with %s:%i (lbrynet v%s)",
address[0], address[1], old_protocol_errors[error_datagram.response])
def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None:
try:
message = decode_datagram(datagram)
except (ValueError, TypeError):
self.peer_manager.report_failure(address[0], address[1])
log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode())
return
if isinstance(message, RequestDatagram):
self.handle_request_datagram(address, message)
elif isinstance(message, ErrorDatagram):
self.handle_error_datagram(address, message)
else:
assert isinstance(message, ResponseDatagram), "sanity"
self.handle_response_datagram(address, message)
async def send_request(self, peer: 'KademliaPeer', request: RequestDatagram) -> ResponseDatagram:
self._send(peer, request)
response_fut = self.sent_messages[request.rpc_id][1]
try:
response = await asyncio.wait_for(response_fut, self.rpc_timeout)
self.peer_manager.report_last_replied(peer.address, peer.udp_port)
return response
except asyncio.CancelledError:
if not response_fut.done():
response_fut.cancel()
raise
except (asyncio.TimeoutError, RemoteException):
self.peer_manager.report_failure(peer.address, peer.udp_port)
if self.peer_manager.peer_is_good(peer) is False:
self.remove_peer(peer)
raise
def send_response(self, peer: 'KademliaPeer', response: ResponseDatagram):
self._send(peer, response)
def send_error(self, peer: 'KademliaPeer', error: ErrorDatagram):
self._send(peer, error)
def _send(self, peer: 'KademliaPeer', message: typing.Union[RequestDatagram, ResponseDatagram,
ErrorDatagram]):
if not self.transport or self.transport.is_closing():
raise TransportNotConnected()
data = message.bencode()
if len(data) > constants.msg_size_limit:
log.exception("unexpected: %i vs %i", len(data), constants.msg_size_limit)
raise ValueError()
if isinstance(message, (RequestDatagram, ResponseDatagram)):
assert message.node_id == self.node_id, message
if isinstance(message, RequestDatagram):
assert self.node_id != peer.node_id
def pop_from_sent_messages(_):
if message.rpc_id in self.sent_messages:
self.sent_messages.pop(message.rpc_id)
if isinstance(message, RequestDatagram):
response_fut = self.loop.create_future()
response_fut.add_done_callback(pop_from_sent_messages)
self.sent_messages[message.rpc_id] = (peer, response_fut, message)
try:
self.transport.sendto(data, (peer.address, peer.udp_port))
except OSError as err:
# TODO: handle ENETUNREACH
if err.errno == socket.EWOULDBLOCK:
# i'm scared this may swallow important errors, but i get a million of these
# on Linux and it doesn't seem to affect anything -grin
log.warning("Can't send data to dht: EWOULDBLOCK")
else:
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
len(data), peer.address, peer.udp_port, str(err), err.errno)
if isinstance(message, RequestDatagram):
self.sent_messages[message.rpc_id][1].set_exception(err)
else:
raise err
if isinstance(message, RequestDatagram):
self.peer_manager.report_last_sent(peer.address, peer.udp_port)
elif isinstance(message, ErrorDatagram):
self.peer_manager.report_failure(peer.address, peer.udp_port)
def change_token(self):
self.old_token_secret = self.token_secret
self.token_secret = constants.generate_id()
def make_token(self, compact_ip):
return constants.digest(self.token_secret + compact_ip)
def verify_token(self, token, compact_ip):
h = constants.hash_class()
h.update(self.token_secret + compact_ip)
if self.old_token_secret and not token == h.digest(): # TODO: why should we be accepting the previous token?
h = constants.hash_class()
h.update(self.old_token_secret + compact_ip)
if not token == h.digest():
return False
return True
async def store_to_peer(self, hash_value: bytes, peer: 'KademliaPeer') -> typing.Tuple[bytes, bool]:
async def __store():
res = await self.get_rpc_peer(peer).store(hash_value)
if res != b"OK":
raise ValueError(res)
log.debug("Stored %s to %s", binascii.hexlify(hash_value).decode()[:8], peer)
return peer.node_id, True
try:
return await __store()
except asyncio.TimeoutError:
log.debug("Timeout while storing blob_hash %s at %s", binascii.hexlify(hash_value).decode()[:8], peer)
except ValueError as err:
log.error("Unexpected response: %s" % err)
except Exception as err:
if 'Invalid token' in str(err):
self.peer_manager.clear_token(peer.node_id)
try:
return await __store()
except:
return peer.node_id, False
else:
log.exception("Unexpected error while storing blob_hash")
return peer.node_id, False
def _write(self, data: bytes, address: typing.Tuple[str, int]):
if self.transport:
try:
self.transport.sendto(data, address)
except OSError as err:
if err.errno == socket.EWOULDBLOCK:
# i'm scared this may swallow important errors, but i get a million of these
# on Linux and it doesn't seem to affect anything -grin
log.warning("Can't send data to dht: EWOULDBLOCK")
# elif err.errno == socket.ENETUNREACH:
# # this should probably try to retransmit when the network connection is back
# log.error("Network is unreachable")
else:
log.error("DHT socket error sending %i bytes to %s:%i - %s (code %i)",
len(data), address[0], address[1], str(err), err.errno)
raise err
else:
raise TransportNotConnected()
-319
View File
@@ -1,319 +0,0 @@
import asyncio
import random
import logging
import typing
import itertools
from lbrynet.dht import constants
from lbrynet.dht.protocol.distance import Distance
if typing.TYPE_CHECKING:
from lbrynet.dht.peer import KademliaPeer, PeerManager
log = logging.getLogger(__name__)
class KBucket:
""" Description - later
"""
def __init__(self, peer_manager: 'PeerManager', range_min: int, range_max: int, node_id: bytes):
"""
@param range_min: The lower boundary for the range in the n-bit ID
space covered by this k-bucket
@param range_max: The upper boundary for the range in the ID space
covered by this k-bucket
"""
self._peer_manager = peer_manager
self.last_accessed = 0
self.range_min = range_min
self.range_max = range_max
self.peers: typing.List['KademliaPeer'] = []
self._node_id = node_id
self._distance_to_self = Distance(node_id)
def add_peer(self, peer: 'KademliaPeer') -> bool:
""" Add contact to _contact list in the right order. This will move the
contact to the end of the k-bucket if it is already present.
@raise kademlia.kbucket.BucketFull: Raised when the bucket is full and
the contact isn't in the bucket
already
@param peer: The contact to add
@type peer: dht.contact._Contact
"""
if peer in self.peers:
# Move the existing contact to the end of the list
# - using the new contact to allow add-on data
# (e.g. optimization-specific stuff) to pe updated as well
self.peers.remove(peer)
self.peers.append(peer)
return True
else:
for i in range(len(self.peers)):
p = self.peers[i]
if p.node_id == peer.node_id:
self.peers.remove(p)
self.peers.append(peer)
return True
if len(self.peers) < constants.k:
self.peers.append(peer)
return True
else:
return False
# raise BucketFull("No space in bucket to insert contact")
def get_peer(self, node_id: bytes) -> 'KademliaPeer':
for peer in self.peers:
if peer.node_id == node_id:
return peer
raise IndexError(node_id)
def get_peers(self, count=-1, exclude_contact=None, sort_distance_to=None) -> typing.List['KademliaPeer']:
""" Returns a list containing up to the first count number of contacts
@param count: The amount of contacts to return (if 0 or less, return
all contacts)
@type count: int
@param exclude_contact: A node node_id to exclude; if this contact is in
the list of returned values, it will be
discarded before returning. If a C{str} is
passed as this argument, it must be the
contact's ID.
@type exclude_contact: str
@param sort_distance_to: Sort distance to the node_id, defaulting to the parent node node_id. If False don't
sort the contacts
@raise IndexError: If the number of requested contacts is too large
@return: Return up to the first count number of contacts in a list
If no contacts are present an empty is returned
@rtype: list
"""
peers = [peer for peer in self.peers if peer.node_id != exclude_contact]
# Return all contacts in bucket
if count <= 0:
count = len(peers)
# Get current contact number
current_len = len(peers)
# If count greater than k - return only k contacts
if count > constants.k:
count = constants.k
if not current_len:
return peers
if sort_distance_to is False:
pass
else:
sort_distance_to = sort_distance_to or self._node_id
peers.sort(key=lambda c: Distance(sort_distance_to)(c.node_id))
return peers[:min(current_len, count)]
def get_bad_or_unknown_peers(self) -> typing.List['KademliaPeer']:
peer = self.get_peers(sort_distance_to=False)
return [
peer for peer in peer
if self._peer_manager.contact_triple_is_good(peer.node_id, peer.address, peer.udp_port) is not True
]
def remove_peer(self, peer: 'KademliaPeer') -> None:
self.peers.remove(peer)
def key_in_range(self, key: bytes) -> bool:
""" Tests whether the specified key (i.e. node ID) is in the range
of the n-bit ID space covered by this k-bucket (in otherwords, it
returns whether or not the specified key should be placed in this
k-bucket)
@param key: The key to test
@type key: str or int
@return: C{True} if the key is in this k-bucket's range, or C{False}
if not.
@rtype: bool
"""
return self.range_min <= self._distance_to_self(key) < self.range_max
def __len__(self) -> int:
return len(self.peers)
def __contains__(self, item) -> bool:
return item in self.peers
class TreeRoutingTable:
""" This class implements a routing table used by a Node class.
The Kademlia routing table is a binary tree whose leaves are k-buckets,
where each k-bucket contains nodes with some common prefix of their IDs.
This prefix is the k-bucket's position in the binary tree; it therefore
covers some range of ID values, and together all of the k-buckets cover
the entire n-bit ID (or key) space (with no overlap).
@note: In this implementation, nodes in the tree (the k-buckets) are
added dynamically, as needed; this technique is described in the 13-page
version of the Kademlia paper, in section 2.4. It does, however, use the
ping RPC-based k-bucket eviction algorithm described in section 2.2 of
that paper.
"""
def __init__(self, loop: asyncio.BaseEventLoop, peer_manager: 'PeerManager', parent_node_id: bytes,
split_buckets_under_index: int = constants.split_buckets_under_index):
self._loop = loop
self._peer_manager = peer_manager
self._parent_node_id = parent_node_id
self._split_buckets_under_index = split_buckets_under_index
self.buckets: typing.List[KBucket] = [
KBucket(
self._peer_manager, range_min=0, range_max=2 ** constants.hash_bits, node_id=self._parent_node_id
)
]
def get_peers(self) -> typing.List['KademliaPeer']:
return list(itertools.chain.from_iterable(map(lambda bucket: bucket.peers, self.buckets)))
def should_split(self, bucket_index: int, to_add: bytes) -> bool:
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
if bucket_index < self._split_buckets_under_index:
return True
contacts = self.get_peers()
distance = Distance(self._parent_node_id)
contacts.sort(key=lambda c: distance(c.node_id))
kth_contact = contacts[-1] if len(contacts) < constants.k else contacts[constants.k - 1]
return distance(to_add) < distance(kth_contact.node_id)
def find_close_peers(self, key: bytes, count: typing.Optional[int] = None,
sender_node_id: typing.Optional[bytes] = None) -> typing.List['KademliaPeer']:
exclude = [self._parent_node_id]
if sender_node_id:
exclude.append(sender_node_id)
count = count or constants.k
distance = Distance(key)
contacts = self.get_peers()
contacts = [c for c in contacts if c.node_id not in exclude]
if contacts:
contacts.sort(key=lambda c: distance(c.node_id))
return contacts[:min(count, len(contacts))]
return []
def get_peer(self, contact_id: bytes) -> 'KademliaPeer':
"""
@raise IndexError: No contact with the specified contact ID is known
by this node
"""
return self.buckets[self.kbucket_index(contact_id)].get_peer(contact_id)
def get_refresh_list(self, start_index: int = 0, force: bool = False) -> typing.List[bytes]:
bucket_index = start_index
refresh_ids = []
now = int(self._loop.time())
for bucket in self.buckets[start_index:]:
if force or now - bucket.last_accessed >= constants.refresh_interval:
to_search = self.midpoint_id_in_bucket_range(bucket_index)
refresh_ids.append(to_search)
bucket_index += 1
return refresh_ids
def remove_peer(self, peer: 'KademliaPeer') -> None:
if not peer.node_id:
return
bucket_index = self.kbucket_index(peer.node_id)
try:
self.buckets[bucket_index].remove_peer(peer)
except ValueError:
return
def touch_kbucket(self, key: bytes) -> None:
self.touch_kbucket_by_index(self.kbucket_index(key))
def touch_kbucket_by_index(self, bucket_index: int):
self.buckets[bucket_index].last_accessed = int(self._loop.time())
def kbucket_index(self, key: bytes) -> int:
i = 0
for bucket in self.buckets:
if bucket.key_in_range(key):
return i
else:
i += 1
return i
def random_id_in_bucket_range(self, bucket_index: int) -> bytes:
random_id = int(random.randrange(self.buckets[bucket_index].range_min, self.buckets[bucket_index].range_max))
return Distance(
self._parent_node_id
)(random_id.to_bytes(constants.hash_length, 'big')).to_bytes(constants.hash_length, 'big')
def midpoint_id_in_bucket_range(self, bucket_index: int) -> bytes:
half = int((self.buckets[bucket_index].range_max - self.buckets[bucket_index].range_min) // 2)
return Distance(self._parent_node_id)(
int(self.buckets[bucket_index].range_min + half).to_bytes(constants.hash_length, 'big')
).to_bytes(constants.hash_length, 'big')
def split_bucket(self, old_bucket_index: int) -> None:
""" Splits the specified k-bucket into two new buckets which together
cover the same range in the key/ID space
@param old_bucket_index: The index of k-bucket to split (in this table's
list of k-buckets)
@type old_bucket_index: int
"""
# Resize the range of the current (old) k-bucket
old_bucket = self.buckets[old_bucket_index]
split_point = old_bucket.range_max - (old_bucket.range_max - old_bucket.range_min) // 2
# Create a new k-bucket to cover the range split off from the old bucket
new_bucket = KBucket(self._peer_manager, split_point, old_bucket.range_max, self._parent_node_id)
old_bucket.range_max = split_point
# Now, add the new bucket into the routing table tree
self.buckets.insert(old_bucket_index + 1, new_bucket)
# Finally, copy all nodes that belong to the new k-bucket into it...
for contact in old_bucket.peers:
if new_bucket.key_in_range(contact.node_id):
new_bucket.add_peer(contact)
# ...and remove them from the old bucket
for contact in new_bucket.peers:
old_bucket.remove_peer(contact)
def join_buckets(self):
if len(self.buckets) == 1:
return
to_pop = [i for i, bucket in enumerate(self.buckets) if not len(bucket)]
if not to_pop:
return
log.info("join buckets %i", len(to_pop))
bucket_index_to_pop = to_pop[0]
assert len(self.buckets[bucket_index_to_pop]) == 0
can_go_lower = bucket_index_to_pop - 1 >= 0
can_go_higher = bucket_index_to_pop + 1 < len(self.buckets)
assert can_go_higher or can_go_lower
bucket = self.buckets[bucket_index_to_pop]
if can_go_lower and can_go_higher:
midpoint = ((bucket.range_max - bucket.range_min) // 2) + bucket.range_min
self.buckets[bucket_index_to_pop - 1].range_max = midpoint - 1
self.buckets[bucket_index_to_pop + 1].range_min = midpoint
elif can_go_lower:
self.buckets[bucket_index_to_pop - 1].range_max = bucket.range_max
elif can_go_higher:
self.buckets[bucket_index_to_pop + 1].range_min = bucket.range_min
self.buckets.remove(bucket)
return self.join_buckets()
def contact_in_routing_table(self, address_tuple: typing.Tuple[str, int]) -> bool:
for bucket in self.buckets:
for contact in bucket.get_peers(sort_distance_to=False):
if address_tuple[0] == contact.address and address_tuple[1] == contact.udp_port:
return True
return False
def buckets_with_contacts(self) -> int:
count = 0
for bucket in self.buckets:
if len(bucket):
count += 1
return count
+320
View File
@@ -0,0 +1,320 @@
# This library is free software, distributed under the terms of
# the GNU Lesser General Public License Version 3, or any later version.
# See the COPYING file included in this archive
#
# The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net
import random
import logging
from twisted.internet import defer
from lbrynet.dht import constants, kbucket
from lbrynet.dht.error import TimeoutError
from lbrynet.dht.distance import Distance
log = logging.getLogger(__name__)
class TreeRoutingTable:
""" This class implements a routing table used by a Node class.
The Kademlia routing table is a binary tree whFose leaves are k-buckets,
where each k-bucket contains nodes with some common prefix of their IDs.
This prefix is the k-bucket's position in the binary tree; it therefore
covers some range of ID values, and together all of the k-buckets cover
the entire n-bit ID (or key) space (with no overlap).
@note: In this implementation, nodes in the tree (the k-buckets) are
added dynamically, as needed; this technique is described in the 13-page
version of the Kademlia paper, in section 2.4. It does, however, use the
C{PING} RPC-based k-bucket eviction algorithm described in section 2.2 of
that paper.
"""
#implements(IRoutingTable)
def __init__(self, parentNodeID, getTime=None):
"""
@param parentNodeID: The n-bit node ID of the node to which this
routing table belongs
@type parentNodeID: str
"""
# Create the initial (single) k-bucket covering the range of the entire n-bit ID space
self._parentNodeID = parentNodeID
self._buckets = [kbucket.KBucket(rangeMin=0, rangeMax=2 ** constants.key_bits, node_id=self._parentNodeID)]
if not getTime:
from twisted.internet import reactor
getTime = reactor.seconds
self._getTime = getTime
self._ongoing_replacements = set()
def get_contacts(self):
contacts = []
for i in range(len(self._buckets)):
for contact in self._buckets[i]._contacts:
contacts.append(contact)
return contacts
def _shouldSplit(self, bucketIndex, toAdd):
# https://stackoverflow.com/questions/32129978/highly-unbalanced-kademlia-routing-table/32187456#32187456
if self._buckets[bucketIndex].keyInRange(self._parentNodeID):
return True
contacts = self.get_contacts()
distance = Distance(self._parentNodeID)
contacts.sort(key=lambda c: distance(c.id))
kth_contact = contacts[-1] if len(contacts) < constants.k else contacts[constants.k-1]
return distance(toAdd) < distance(kth_contact.id)
def addContact(self, contact):
""" Add the given contact to the correct k-bucket; if it already
exists, its status will be updated
@param contact: The contact to add to this node's k-buckets
@type contact: kademlia.contact.Contact
@rtype: defer.Deferred
"""
if contact.id == self._parentNodeID:
return defer.succeed(None)
bucketIndex = self._kbucketIndex(contact.id)
try:
self._buckets[bucketIndex].addContact(contact)
except kbucket.BucketFull:
# The bucket is full; see if it can be split (by checking if its range includes the host node's id)
if self._shouldSplit(bucketIndex, contact.id):
self._splitBucket(bucketIndex)
# Retry the insertion attempt
return self.addContact(contact)
else:
# We can't split the k-bucket
#
# The 13 page kademlia paper specifies that the least recently contacted node in the bucket
# shall be pinged. If it fails to reply it is replaced with the new contact. If the ping is successful
# the new contact is ignored and not added to the bucket (sections 2.2 and 2.4).
#
# A reasonable extension to this is BEP 0005, which extends the above:
#
# Not all nodes that we learn about are equal. Some are "good" and some are not.
# Many nodes using the DHT are able to send queries and receive responses,
# but are not able to respond to queries from other nodes. It is important that
# each node's routing table must contain only known good nodes. A good node is
# a node has responded to one of our queries within the last 15 minutes. A node
# is also good if it has ever responded to one of our queries and has sent us a
# query within the last 15 minutes. After 15 minutes of inactivity, a node becomes
# questionable. Nodes become bad when they fail to respond to multiple queries
# in a row. Nodes that we know are good are given priority over nodes with unknown status.
#
# When there are bad or questionable nodes in the bucket, the least recent is selected for
# potential replacement (BEP 0005). When all nodes in the bucket are fresh, the head (least recent)
# contact is selected as described in section 2.2 of the kademlia paper. In both cases the new contact
# is ignored if the pinged node replies.
def replaceContact(failure, deadContact):
"""
Callback for the deferred PING RPC to see if the node to be replaced in the k-bucket is still
responding
@type failure: twisted.python.failure.Failure
"""
failure.trap(TimeoutError)
log.debug("Replacing dead contact in bucket %i: %s:%i (%s) with %s:%i (%s)", bucketIndex,
deadContact.address, deadContact.port, deadContact.log_id(), contact.address,
contact.port, contact.log_id())
try:
self._buckets[bucketIndex].removeContact(deadContact)
except ValueError:
# The contact has already been removed (probably due to a timeout)
pass
return self.addContact(contact)
not_good_contacts = self._buckets[bucketIndex].getBadOrUnknownContacts()
if not_good_contacts:
to_replace = not_good_contacts[0]
else:
to_replace = self._buckets[bucketIndex]._contacts[0]
if to_replace not in self._ongoing_replacements:
log.debug("pinging %s:%s", to_replace.address, to_replace.port)
self._ongoing_replacements.add(to_replace)
df = to_replace.ping()
df.addErrback(replaceContact, to_replace)
df.addBoth(lambda _: self._ongoing_replacements.remove(to_replace))
else:
df = defer.succeed(None)
return df
else:
self.touchKBucketByIndex(bucketIndex)
return defer.succeed(None)
def findCloseNodes(self, key, count=None, sender_node_id=None):
""" Finds a number of known nodes closest to the node/value with the
specified key.
@param key: the n-bit key (i.e. the node or value ID) to search for
@type key: str
@param count: the amount of contacts to return, default of k (8)
@type count: int
@param sender_node_id: Used during RPC, this is be the sender's Node ID
Whatever ID is passed in the parameter will get
excluded from the list of returned contacts.
@type sender_node_id: str
@return: A list of node contacts (C{kademlia.contact.Contact instances})
closest to the specified key.
This method will return C{k} (or C{count}, if specified)
contacts if at all possible; it will only return fewer if the
node is returning all of the contacts that it knows of.
@rtype: list
"""
exclude = [self._parentNodeID]
if sender_node_id:
exclude.append(sender_node_id)
if key in exclude:
exclude.remove(key)
count = count or constants.k
distance = Distance(key)
contacts = self.get_contacts()
contacts = [c for c in contacts if c.id not in exclude]
contacts.sort(key=lambda c: distance(c.id))
return contacts[:min(count, len(contacts))]
def getContact(self, contactID):
""" Returns the (known) contact with the specified node ID
@raise ValueError: No contact with the specified contact ID is known
by this node
"""
bucketIndex = self._kbucketIndex(contactID)
return self._buckets[bucketIndex].getContact(contactID)
def getRefreshList(self, startIndex=0, force=False):
""" Finds all k-buckets that need refreshing, starting at the
k-bucket with the specified index, and returns IDs to be searched for
in order to refresh those k-buckets
@param startIndex: The index of the bucket to start refreshing at;
this bucket and those further away from it will
be refreshed. For example, when joining the
network, this node will set this to the index of
the bucket after the one containing it's closest
neighbour.
@type startIndex: index
@param force: If this is C{True}, all buckets (in the specified range)
will be refreshed, regardless of the time they were last
accessed.
@type force: bool
@return: A list of node ID's that the parent node should search for
in order to refresh the routing Table
@rtype: list
"""
bucketIndex = startIndex
refreshIDs = []
now = int(self._getTime())
for bucket in self._buckets[startIndex:]:
if force or now - bucket.lastAccessed >= constants.refreshTimeout:
searchID = self.midpoint_id_in_bucket_range(bucketIndex)
refreshIDs.append(searchID)
bucketIndex += 1
return refreshIDs
def removeContact(self, contact):
"""
Remove the contact from the routing table
@param contact: The contact to remove
@type contact: dht.contact._Contact
"""
bucketIndex = self._kbucketIndex(contact.id)
try:
self._buckets[bucketIndex].removeContact(contact)
except ValueError:
return
def touchKBucket(self, key):
""" Update the "last accessed" timestamp of the k-bucket which covers
the range containing the specified key in the key/ID space
@param key: A key in the range of the target k-bucket
@type key: str
"""
self.touchKBucketByIndex(self._kbucketIndex(key))
def touchKBucketByIndex(self, bucketIndex):
self._buckets[bucketIndex].lastAccessed = int(self._getTime())
def _kbucketIndex(self, key):
""" Calculate the index of the k-bucket which is responsible for the
specified key (or ID)
@param key: The key for which to find the appropriate k-bucket index
@type key: str
@return: The index of the k-bucket responsible for the specified key
@rtype: int
"""
i = 0
for bucket in self._buckets:
if bucket.keyInRange(key):
return i
else:
i += 1
return i
def random_id_in_bucket_range(self, bucketIndex):
""" Returns a random ID in the specified k-bucket's range
@param bucketIndex: The index of the k-bucket to use
@type bucketIndex: int
"""
random_id = int(random.randrange(self._buckets[bucketIndex].rangeMin, self._buckets[bucketIndex].rangeMax))
return random_id.to_bytes(constants.key_bits // 8, 'big')
def midpoint_id_in_bucket_range(self, bucketIndex):
""" Returns the middle ID in the specified k-bucket's range
@param bucketIndex: The index of the k-bucket to use
@type bucketIndex: int
"""
half = int((self._buckets[bucketIndex].rangeMax - self._buckets[bucketIndex].rangeMin) // 2)
return int(self._buckets[bucketIndex].rangeMin + half).to_bytes(constants.key_bits // 8, 'big')
def _splitBucket(self, oldBucketIndex):
""" Splits the specified k-bucket into two new buckets which together
cover the same range in the key/ID space
@param oldBucketIndex: The index of k-bucket to split (in this table's
list of k-buckets)
@type oldBucketIndex: int
"""
# Resize the range of the current (old) k-bucket
oldBucket = self._buckets[oldBucketIndex]
splitPoint = oldBucket.rangeMax - (oldBucket.rangeMax - oldBucket.rangeMin) / 2
# Create a new k-bucket to cover the range split off from the old bucket
newBucket = kbucket.KBucket(splitPoint, oldBucket.rangeMax, self._parentNodeID)
oldBucket.rangeMax = splitPoint
# Now, add the new bucket into the routing table tree
self._buckets.insert(oldBucketIndex + 1, newBucket)
# Finally, copy all nodes that belong to the new k-bucket into it...
for contact in oldBucket._contacts:
if newBucket.keyInRange(contact.id):
newBucket.addContact(contact)
# ...and remove them from the old bucket
for contact in newBucket._contacts:
oldBucket.removeContact(contact)
def contactInRoutingTable(self, address_tuple):
for bucket in self._buckets:
for contact in bucket.getContacts(sort_distance_to=False):
if address_tuple[0] == contact.address and address_tuple[1] == contact.port:
return True
return False
def bucketsWithContacts(self):
count = 0
for bucket in self._buckets:
if len(bucket):
count += 1
return count
-178
View File
@@ -1,178 +0,0 @@
import typing
from functools import reduce
from lbrynet.dht import constants
from lbrynet.dht.serialization.bencoding import bencode, bdecode
REQUEST_TYPE = 0
RESPONSE_TYPE = 1
ERROR_TYPE = 2
class KademliaDatagramBase:
"""
field names are used to unwrap/wrap the argument names to index integers that replace them in a datagram
all packets have an argument dictionary when bdecoded starting with {0: <int>, 1: <bytes>, 2: <bytes>, ...}
these correspond to the packet_type, rpc_id, and node_id args
"""
fields = [
'packet_type',
'rpc_id',
'node_id'
]
expected_packet_type = -1
def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes):
self.packet_type = packet_type
if self.expected_packet_type != packet_type:
raise ValueError(f"invalid packet type: {packet_type}, expected {self.expected_packet_type}")
if len(rpc_id) != constants.rpc_id_length:
raise ValueError(f"invalid rpc node_id: {len(rpc_id)} bytes (expected 20)")
if not len(node_id) == constants.hash_length:
raise ValueError(f"invalid node node_id: {len(node_id)} bytes (expected 48)")
self.rpc_id = rpc_id
self.node_id = node_id
def bencode(self) -> bytes:
return bencode({
i: getattr(self, k) for i, k in enumerate(self.fields)
})
class RequestDatagram(KademliaDatagramBase):
fields = [
'packet_type',
'rpc_id',
'node_id',
'method',
'args'
]
expected_packet_type = REQUEST_TYPE
def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes, method: bytes,
args: typing.Optional[typing.List] = None):
super().__init__(packet_type, rpc_id, node_id)
self.method = method
self.args = args or []
if not self.args:
self.args.append({})
if isinstance(self.args[-1], dict):
self.args[-1][b'protocolVersion'] = 1
else:
self.args.append({b'protocolVersion': 1})
@classmethod
def make_ping(cls, from_node_id: bytes, rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'ping')
@classmethod
def make_store(cls, from_node_id: bytes, blob_hash: bytes, token: bytes, port: int,
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
if len(blob_hash) != constants.hash_bits // 8:
raise ValueError(f"invalid blob hash length: {len(blob_hash)}")
if not 0 < port < 65536:
raise ValueError(f"invalid port: {port}")
if len(token) != constants.hash_bits // 8:
raise ValueError(f"invalid token length: {len(token)}")
store_args = [blob_hash, token, port, from_node_id, 0]
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'store', store_args)
@classmethod
def make_find_node(cls, from_node_id: bytes, key: bytes,
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
if len(key) != constants.hash_bits // 8:
raise ValueError(f"invalid key length: {len(key)}")
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findNode', [key])
@classmethod
def make_find_value(cls, from_node_id: bytes, key: bytes,
rpc_id: typing.Optional[bytes] = None) -> 'RequestDatagram':
rpc_id = rpc_id or constants.generate_id()[:constants.rpc_id_length]
if len(key) != constants.hash_bits // 8:
raise ValueError(f"invalid key length: {len(key)}")
return cls(REQUEST_TYPE, rpc_id, from_node_id, b'findValue', [key])
class ResponseDatagram(KademliaDatagramBase):
fields = [
'packet_type',
'rpc_id',
'node_id',
'response'
]
expected_packet_type = RESPONSE_TYPE
def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes, response):
super().__init__(packet_type, rpc_id, node_id)
self.response = response
class ErrorDatagram(KademliaDatagramBase):
fields = [
'packet_type',
'rpc_id',
'node_id',
'exception_type',
'response',
]
expected_packet_type = ERROR_TYPE
def __init__(self, packet_type: int, rpc_id: bytes, node_id: bytes, exception_type: bytes, response: bytes):
super().__init__(packet_type, rpc_id, node_id)
self.exception_type = exception_type.decode()
self.response = response.decode()
def decode_datagram(datagram: bytes) -> typing.Union[RequestDatagram, ResponseDatagram, ErrorDatagram]:
msg_types = {
REQUEST_TYPE: RequestDatagram,
RESPONSE_TYPE: ResponseDatagram,
ERROR_TYPE: ErrorDatagram
}
primitive: typing.Dict = bdecode(datagram)
if primitive[0] in [REQUEST_TYPE, ERROR_TYPE, RESPONSE_TYPE]: # pylint: disable=unsubscriptable-object
datagram_type = primitive[0] # pylint: disable=unsubscriptable-object
else:
raise ValueError("invalid datagram type")
datagram_class = msg_types[datagram_type]
return datagram_class(**{
k: primitive[i] # pylint: disable=unsubscriptable-object
for i, k in enumerate(datagram_class.fields)
if i in primitive # pylint: disable=unsupported-membership-test
}
)
def make_compact_ip(address: str) -> bytearray:
compact_ip = reduce(lambda buff, x: buff + bytearray([int(x)]), address.split('.'), bytearray())
if len(compact_ip) != 4:
raise ValueError(f"invalid IPv4 length")
return compact_ip
def make_compact_address(node_id: bytes, address: str, port: int) -> bytearray:
compact_ip = make_compact_ip(address)
if not 0 < port < 65536:
raise ValueError(f'Invalid port: {port}')
if len(node_id) != constants.hash_bits // 8:
raise ValueError(f"invalid node node_id length")
return compact_ip + port.to_bytes(2, 'big') + node_id
def decode_compact_address(compact_address: bytes) -> typing.Tuple[bytes, str, int]:
address = "{}.{}.{}.{}".format(*compact_address[:4])
port = int.from_bytes(compact_address[4:6], 'big')
node_id = compact_address[6:]
if not 0 < port < 65536:
raise ValueError(f'Invalid port: {port}')
if len(node_id) != constants.hash_bits // 8:
raise ValueError(f"invalid node node_id length")
return node_id, address, port
+115 -201
View File
@@ -1,20 +1,19 @@
import os
import sys
import signal
import pathlib
import json
import asyncio
import argparse
import logging
import logging.handlers
from docopt import docopt
from textwrap import dedent
import aiohttp
from aiohttp.web import GracefulExit
from lbrynet.extras.compat import force_asyncioreactor_install
force_asyncioreactor_install()
from lbrynet import __name__ as lbrynet_name, __version__ as lbrynet_version
from lbrynet import log_support, __name__ as lbrynet_name, __version__ as lbrynet_version
from lbrynet.extras.daemon.loggly_handler import get_loggly_handler
from lbrynet.conf import Config, CLIConfig
from lbrynet.utils import check_connection
from lbrynet.extras.daemon.Daemon import Daemon
log = logging.getLogger(lbrynet_name)
@@ -25,7 +24,7 @@ def display(data):
print(json.dumps(data, indent=2))
async def execute_command(conf, method, params, callback=display):
async def execute_command(conf, method, params):
async with aiohttp.ClientSession() as session:
try:
message = {'method': method, 'params': params}
@@ -33,15 +32,55 @@ async def execute_command(conf, method, params, callback=display):
try:
data = await resp.json()
if 'result' in data:
return callback(data['result'])
display(data['result'])
elif 'error' in data:
return callback(data['error'])
if 'message' in data['error']:
display(data['error']['message'])
else:
display(data['error'])
except Exception as e:
log.exception('Could not process response from server:', exc_info=e)
except aiohttp.ClientConnectionError:
print("Could not connect to daemon. Are you sure it's running?")
def print_help():
print(dedent("""
NAME
lbrynet - LBRY command line client.
USAGE
lbrynet [--data_dir=<blob and database directory>] [--wallet_dir=<wallet directory>]
[--download_dir=<downloads directory>] <command> [<args>]
EXAMPLES
lbrynet start # starts the daemon and listens for jsonrpc commands
lbrynet help # display this message
lbrynet help <command_name> # get help for a command(doesn't need the daemon to be running)
lbrynet commands # list available commands
lbrynet status # get the running status of the daemon
lbrynet resolve what # resolve a name
lbrynet --wallet_dir=~/wallet2 start # start the daemon using an alternative wallet directory
lbrynet --data_dir=~/lbry start # start the daemon using an alternative data directory
lbrynet --data_dir=~/lbry <command_name> # run a command on a daemon using an alternative data directory,
# which can contain a full daemon_settings.yml config file.
# Note: since the daemon is what runs the wallet and
# downloads files, only the --data_dir setting is needed when
# running commands. The wallet_dir and download_dir would only
# by used when starting the daemon.
"""))
def print_help_for_command(command):
fn = Daemon.callable_methods.get(command)
if fn:
print(dedent(fn.__doc__))
else:
print("Invalid command name")
def normalize_value(x, key=None):
if not isinstance(x, str):
return x
@@ -76,113 +115,21 @@ def set_kwargs(parsed_args):
return kwargs
def split_subparser_argument(parent, original, name, condition):
new_sub_parser = argparse._SubParsersAction(
original.option_strings,
original._prog_prefix,
original._parser_class,
metavar=original.metavar
)
new_sub_parser._name_parser_map = original._name_parser_map
new_sub_parser._choices_actions = [
a for a in original._choices_actions if condition(original._name_parser_map[a.dest])
]
group = argparse._ArgumentGroup(parent, name)
group._group_actions = [new_sub_parser]
return group
class ArgumentParser(argparse.ArgumentParser):
def __init__(self, *args, group_name=None, **kwargs):
super().__init__(*args, formatter_class=HelpFormatter, add_help=False, **kwargs)
self.add_argument(
'--help', dest='help', action='store_true', default=False,
help='Show this help message and exit.'
)
self._optionals.title = 'Options'
if group_name is None:
self.epilog = (
f"Run 'lbrynet COMMAND --help' for more information on a command or group."
)
else:
self.epilog = (
f"Run 'lbrynet {group_name} COMMAND --help' for more information on a command."
)
self.set_defaults(group=group_name, group_parser=self)
def format_help(self):
formatter = self._get_formatter()
formatter.add_usage(
self.usage, self._actions, self._mutually_exclusive_groups
)
formatter.add_text(self.description)
# positionals, optionals and user-defined groups
for action_group in self._granular_action_groups:
formatter.start_section(action_group.title)
formatter.add_text(action_group.description)
formatter.add_arguments(action_group._group_actions)
formatter.end_section()
formatter.add_text(self.epilog)
return formatter.format_help()
@property
def _granular_action_groups(self):
if self.prog != 'lbrynet':
yield from self._action_groups
return
yield self._optionals
action: argparse._SubParsersAction = self._positionals._group_actions[0]
yield split_subparser_argument(
self, action, "Grouped Commands", lambda parser: 'group' in parser._defaults
)
yield split_subparser_argument(
self, action, "Commands", lambda parser: 'group' not in parser._defaults
)
def error(self, message):
self.print_help(argparse._sys.stderr)
self.exit(2, '\n'+message+'\n')
class HelpFormatter(argparse.HelpFormatter):
def add_usage(self, usage, actions, groups, prefix='Usage: '):
super().add_usage(
usage, [a for a in actions if a.option_strings != ['--help']], groups, prefix
)
def add_command_parser(parent, command):
subcommand = parent.add_parser(
command['name'],
help=command['doc'].strip().splitlines()[0]
)
subcommand.set_defaults(
api_method_name=command['api_method_name'],
command=command['name'],
doc=command['doc'],
replaced_by=command.get('replaced_by', None)
)
def get_argument_parser():
main = ArgumentParser(
'lbrynet', description='An interface to the LBRY Network.'
)
main = argparse.ArgumentParser('lbrynet', add_help=False)
main.add_argument(
'-v', '--version', dest='cli_version', action="store_true",
'--version', dest='cli_version', action="store_true",
help='Show lbrynet CLI version and exit.'
)
main.set_defaults(group=None, command=None)
CLIConfig.contribute_to_argparse(main)
sub = main.add_subparsers(metavar='COMMAND')
start = sub.add_parser(
'start',
usage='lbrynet start [--config FILE] [--data-dir DIR] [--wallet-dir DIR] [--download-dir DIR] ...',
help='Start LBRY Network interface.'
main.add_argument(
'-h', '--help', dest='help', action="store_true",
help='Show this help message and exit'
)
CLIConfig.contribute_args(main)
sub = main.add_subparsers(dest='command')
help = sub.add_parser('help', help='Detailed help for remote commands.')
help.add_argument('help_command', nargs='*')
start = sub.add_parser('start', help='Start lbrynet server.')
start.add_argument(
'--quiet', dest='quiet', action="store_true",
help='Disable all console output.'
@@ -192,43 +139,28 @@ def get_argument_parser():
help=('Enable debug output. Optionally specify loggers for which debug output '
'should selectively be applied.')
)
start.set_defaults(command='start', start_parser=start)
Config.contribute_to_argparse(start)
Config.contribute_args(start)
api = Daemon.get_api_definitions()
groups = {}
for group_name in sorted(api['groups']):
group_parser = sub.add_parser(group_name, group_name=group_name, help=api['groups'][group_name])
groups[group_name] = group_parser.add_subparsers(metavar='COMMAND')
nicer_order = ['stop', 'get', 'publish', 'resolve']
for command_name in sorted(api['commands']):
if command_name not in nicer_order:
nicer_order.append(command_name)
for command_name in nicer_order:
command = api['commands'][command_name]
if command['group'] is None:
add_command_parser(sub, command)
else:
add_command_parser(groups[command['group']], command)
for group in sorted(api):
group_command = sub.add_parser(group, help=api[group]['doc'])
group_command.set_defaults(group_doc=group_command)
if group in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
continue
commands = group_command.add_subparsers(dest='subcommand')
for command in api[group]['commands']:
commands.add_parser(command['name'], help=command['doc'].strip().splitlines()[0])
for deprecated in Daemon.deprecated_methods:
group_command = sub.add_parser(deprecated)
group_command.add_subparsers(dest='subcommand')
return main
def ensure_directory_exists(path: str):
if not os.path.isdir(path):
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
def main(argv=None):
argv = argv or sys.argv[1:]
parser = get_argument_parser()
args, command_args = parser.parse_known_args(argv)
conf = Config.create_from_arguments(args)
for directory in (conf.data_dir, conf.download_dir, conf.wallet_dir):
ensure_directory_exists(directory)
if args.cli_version:
print(f"{lbrynet_name} {lbrynet_version}")
@@ -236,81 +168,63 @@ def main(argv=None):
elif args.command == 'start':
if args.help:
args.start_parser.print_help()
return 0
default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
file_handler = logging.handlers.RotatingFileHandler(
conf.log_file_path, maxBytes=2097152, backupCount=5
)
file_handler.setFormatter(default_formatter)
log.addHandler(file_handler)
logging.getLogger('torba').addHandler(file_handler)
log_support.configure_logging(conf.log_file_path, not args.quiet, args.verbose)
if not args.quiet:
handler = logging.StreamHandler()
handler.setFormatter(default_formatter)
log.addHandler(handler)
logging.getLogger('torba').addHandler(handler)
logging.getLogger('torba').setLevel(logging.INFO)
logging.getLogger('aioupnp').setLevel(logging.WARNING)
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
loop = asyncio.get_event_loop()
if args.verbose:
log.setLevel(logging.DEBUG)
loop.set_debug(True)
else:
log.setLevel(logging.INFO)
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
log.addHandler(loggly_handler)
log.debug('Final Settings: %s', conf.settings_dict)
log.info("Starting lbrynet-daemon from command line")
daemon = Daemon(conf)
started = False
def __exit():
if started:
daemon.stop_event.set()
else:
raise GracefulExit()
try:
loop.add_signal_handler(signal.SIGINT, __exit)
loop.add_signal_handler(signal.SIGTERM, __exit)
except NotImplementedError:
pass # Not implemented on Windows
try:
loop.run_until_complete(daemon.start())
started = True
loop.run_until_complete(daemon.stop_event.wait())
except (GracefulExit, KeyboardInterrupt):
pass
finally:
loop.run_until_complete(daemon.stop())
if hasattr(loop, 'shutdown_asyncgens'):
loop.run_until_complete(loop.shutdown_asyncgens())
if check_connection():
from twisted.internet import reactor
reactor._asyncioEventloop.create_task(daemon.start())
reactor.run()
else:
log.info("Not connected to internet, unable to start")
elif args.command == 'help':
if args.help_command:
method = '_'.join(args.help_command)
else:
parser.print_help()
return 0
if method not in Daemon.callable_methods:
print('Invalid command name: {method}')
return 1
fn = Daemon.callable_methods[method]
print(fn.__doc__)
elif args.command is not None:
doc = args.doc
api_method_name = args.api_method_name
if args.replaced_by:
print(f"{args.api_method_name} is deprecated, using {args.replaced_by['api_method_name']}.")
doc = args.replaced_by['doc']
api_method_name = args.replaced_by['api_method_name']
if args.help:
print(doc)
if args.command in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
method = args.command
elif args.subcommand is not None:
method = f'{args.command}_{args.subcommand}'
else:
parsed = docopt(doc, command_args)
params = set_kwargs(parsed)
loop = asyncio.get_event_loop()
loop.run_until_complete(execute_command(conf, api_method_name, params))
args.group_doc.print_help()
return 0
elif args.group is not None:
args.group_parser.print_help()
if method in Daemon.deprecated_methods:
new_method = Daemon.deprecated_methods[method].new_command
if new_method is None:
print(f"{method} is permanently deprecated and does not have a replacement command.")
return 0
print(f"{method} is deprecated, using {new_method}.")
method = new_method
fn = Daemon.callable_methods[method]
parsed = docopt(fn.__doc__, command_args)
params = set_kwargs(parsed)
loop = asyncio.get_event_loop()
loop.run_until_complete(execute_command(conf, method, params))
else:
parser.print_help()
+25
View File
@@ -0,0 +1,25 @@
import asyncio
def force_asyncioreactor_install():
import sys
from twisted.internet import asyncioreactor
if 'twisted.internet.reactor' not in sys.modules:
asyncioreactor.install()
else:
from twisted.internet import reactor
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
# pyinstaller hooks install the default reactor before
# any of our code runs, see kivy for similar problem:
# https://github.com/kivy/kivy/issues/4182
del sys.modules['twisted.internet.reactor']
asyncioreactor.install()
def d2f(deferred):
return deferred.asFuture(asyncio.get_event_loop())
def f2d(future):
from twisted.internet import defer
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
+5 -3
View File
@@ -1,5 +1,7 @@
import asyncio
import logging
from twisted.internet import defer
from twisted._threads import AlreadyQuit
from lbrynet.conf import Config
from lbrynet.extras.daemon.ComponentManager import ComponentManager
@@ -55,7 +57,7 @@ class Component(metaclass=ComponentType):
result = await self.start()
self._running = True
return result
except asyncio.CancelledError:
except (defer.CancelledError, AlreadyQuit):
pass
except Exception as err:
log.exception("Error setting up %s", self.component_name or self.__class__.__name__)
@@ -66,7 +68,7 @@ class Component(metaclass=ComponentType):
result = await self.stop()
self._running = False
return result
except asyncio.CancelledError:
except (defer.CancelledError, AlreadyQuit):
pass
except Exception as err:
log.exception("Error stopping %s", self.__class__.__name__)
+31 -13
View File
@@ -1,8 +1,9 @@
import logging
import asyncio
import logging
from lbrynet.p2p.Error import ComponentStartConditionNotMet
from lbrynet.extras.daemon.PeerManager import PeerManager
from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
from lbrynet.conf import Config
from lbrynet.error import ComponentStartConditionNotMet
from lbrynet.dht.peer import PeerManager
log = logging.getLogger(__name__)
@@ -34,16 +35,16 @@ class RequiredCondition(metaclass=RequiredConditionType):
class ComponentManager:
default_component_classes = {}
def __init__(self, conf: Config, analytics_manager=None, skip_components=None,
peer_manager=None, **override_components):
def __init__(self, conf: Config, reactor=None, analytics_manager=None, skip_components=None,
peer_manager=None, peer_finder=None, **override_components):
self.conf = conf
self.skip_components = skip_components or []
self.loop = asyncio.get_event_loop()
self.analytics_manager = analytics_manager
self.reactor = reactor
self.component_classes = {}
self.components = set()
self.started = asyncio.Event(loop=self.loop)
self.peer_manager = peer_manager or PeerManager(asyncio.get_event_loop_policy().get_event_loop())
self.analytics_manager = analytics_manager
self.peer_manager = peer_manager or PeerManager()
self.peer_finder = peer_finder or DHTPeerFinder(self)
for component_name, component_class in self.default_component_classes.items():
if component_name in override_components:
@@ -56,6 +57,7 @@ class ComponentManager:
for component_class in self.component_classes.values():
self.components.add(component_class(self))
self.daemon = None
def evaluate_condition(self, condition_name):
if condition_name not in RegisteredConditions.conditions:
@@ -111,19 +113,35 @@ class ComponentManager:
steps.reverse()
return steps
async def start(self):
async def setup(self, **callbacks):
""" Start Components in sequence sorted by requirements """
for stage in self.sort_components():
for component_name, cb in callbacks.items():
if component_name not in self.component_classes:
if component_name not in self.skip_components:
raise NameError("unknown component: %s" % component_name)
if not callable(cb):
raise ValueError("%s is not callable" % cb)
async def _setup(component):
await component._setup()
if component.component_name in callbacks:
maybe_coro = callbacks[component.component_name](component)
if asyncio.iscoroutine(maybe_coro):
asyncio.create_task(maybe_coro)
stages = self.sort_components()
for stage in stages:
needing_start = [
component._setup() for component in stage if not component.running
_setup(component) for component in stage if not component.running
]
if needing_start:
await asyncio.wait(needing_start)
self.started.set()
async def stop(self):
"""
Stop Components in reversed startup order
:return: (defer.Deferred)
"""
stages = self.sort_components(reverse=True)
for stage in stages:
+267 -153
View File
@@ -1,29 +1,40 @@
import os
import asyncio
import aiohttp
import logging
import treq
import math
import binascii
import typing
from hashlib import sha256
from types import SimpleNamespace
import base58
from twisted.internet import defer, reactor, error
from aioupnp import __version__ as aioupnp_version
from aioupnp.upnp import UPnP
from aioupnp.fault import UPnPError
from lbrynet import utils
import lbrynet.schema
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
from lbrynet.extras.compat import d2f
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
from lbrynet.dht.node import Node
from lbrynet.dht.blob_announcer import BlobAnnouncer
from lbrynet.blob.blob_manager import BlobManager
from lbrynet.blob_exchange.server import BlobServer
from lbrynet.stream.stream_manager import StreamManager
from lbrynet.extras.daemon.Component import Component
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.wallet import LbryWalletManager
from lbrynet.wallet import Network
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.wallet import Network
from lbrynet.utils import generate_id
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
from lbrynet.p2p.RateLimiter import RateLimiter
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory
from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory
log = logging.getLogger(__name__)
@@ -36,10 +47,13 @@ HEADERS_COMPONENT = "blockchain_headers"
WALLET_COMPONENT = "wallet"
DHT_COMPONENT = "dht"
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
STREAM_MANAGER_COMPONENT = "stream_manager"
FILE_MANAGER_COMPONENT = "file_manager"
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
REFLECTOR_COMPONENT = "reflector"
UPNP_COMPONENT = "upnp"
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
RATE_LIMITER_COMPONENT = "rate_limiter"
PAYMENT_RATE_COMPONENT = "payment_rate_manager"
async def gather_dict(tasks: dict):
@@ -50,6 +64,17 @@ async def gather_dict(tasks: dict):
)))
async def get_external_ip(): # used if upnp is disabled or non-functioning
try:
async with aiohttp.ClientSession() as session:
async with session.get("https://api.lbry.io/ip") as resp:
response = await resp.json()
if response['success']:
return response['data']['ip']
except Exception as e:
pass
class DatabaseComponent(Component):
component_name = DATABASE_COMPONENT
@@ -63,11 +88,11 @@ class DatabaseComponent(Component):
@staticmethod
def get_current_db_revision():
return 11
return 9
@property
def revision_filename(self):
return os.path.join(self.conf.data_dir, 'db_revision')
return self.component_manager.daemon.db_revision_file_path
def _write_db_revision_file(self, version_num):
with open(self.revision_filename, mode='w') as db_revision:
@@ -93,7 +118,7 @@ class DatabaseComponent(Component):
from lbrynet.extras.daemon.migrator import dbmigrator
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
await asyncio.get_event_loop().run_in_executor(
None, dbmigrator.migrate_db, self.conf, old_revision, self.get_current_db_revision()
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
)
self._write_db_revision_file(self.get_current_db_revision())
log.info("Finished upgrading the databases.")
@@ -133,32 +158,39 @@ class HeadersComponent(Component):
'download_progress': self._headers_progress_percent
}
async def fetch_headers_from_s3(self):
@defer.inlineCallbacks
def fetch_headers_from_s3(self):
def collector(data, h_file):
h_file.write(data)
local_size = float(h_file.tell())
final_size = float(final_size_after_download)
self._headers_progress_percent = math.ceil(local_size / final_size * 100)
local_header_size = self.local_header_file_size()
resume_header = {"Range": f"bytes={local_header_size}-"}
async with utils.aiohttp_request('get', HEADERS_URL, headers=resume_header) as response:
if response.status == 406 or response.content_length < HEADER_SIZE: # our file is bigger
response = yield treq.get(HEADERS_URL, headers=resume_header)
got_406 = response.code == 406 # our file is bigger
final_size_after_download = response.length + local_header_size
if got_406:
log.warning("s3 is more out of date than we are")
# should have something to download and a final length divisible by the header size
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
s3_height = (final_size_after_download / HEADER_SIZE) - 1
local_height = self.local_header_file_height()
if s3_height > local_height:
if local_header_size:
log.info("Resuming download of %i bytes from s3", response.length)
with open(self.headers_file, "a+b") as headers_file:
yield treq.collect(response, lambda d: collector(d, headers_file))
else:
with open(self.headers_file, "wb") as headers_file:
yield treq.collect(response, lambda d: collector(d, headers_file))
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.", s3_height)
self._check_header_file_integrity()
else:
log.warning("s3 is more out of date than we are")
return
if response.content_length % HEADER_SIZE != 0:
log.warning("s3 appears to have corrupted header")
return
final_size_after_download = response.content_length + local_header_size
write_mode = "wb"
if local_header_size > 0:
log.info("Resuming download of %i bytes from s3", response.content_length)
write_mode = "a+b"
with open(self.headers_file, write_mode) as fd:
while True:
chunk = await response.content.read(512)
if not chunk:
break
fd.write(chunk)
self._headers_progress_percent = math.ceil(
float(fd.tell()) / float(final_size_after_download) * 100
)
log.info("fetched headers from s3, now verifying integrity after download.")
self._check_header_file_integrity()
else:
log.error("invalid size for headers from s3")
def local_header_file_height(self):
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
@@ -227,7 +259,7 @@ class HeadersComponent(Component):
self._downloading_headers = await self.should_download_headers_from_s3()
if self._downloading_headers:
try:
await self.fetch_headers_from_s3()
await d2f(self.fetch_headers_from_s3())
except Exception as err:
log.error("failed to fetch headers from s3: %s", err)
finally:
@@ -251,20 +283,23 @@ class WalletComponent(Component):
async def get_status(self):
if self.wallet_manager and self.running:
local_height = self.wallet_manager.ledger.headers.height
remote_height = await self.wallet_manager.ledger.network.get_server_height()
local_height = self.wallet_manager.network.get_local_height()
remote_height = self.wallet_manager.network.get_server_height()
best_hash = self.wallet_manager.get_best_blockhash()
return {
'blocks': max(local_height, 0),
'blocks_behind': max(remote_height - local_height, 0),
'best_blockhash': best_hash,
'is_encrypted': self.wallet_manager.use_encryption,
'is_encrypted': self.wallet_manager.wallet.use_encryption,
'is_locked': not self.wallet_manager.is_wallet_unlocked,
}
async def start(self):
log.info("Starting torba wallet")
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf)
storage = self.component_manager.get_component(DATABASE_COMPONENT)
lbrynet.schema.BLOCKCHAIN_NAME = self.conf.blockchain_name
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf, storage)
self.wallet_manager.old_db = storage
await self.wallet_manager.start()
async def stop(self):
@@ -278,36 +313,30 @@ class BlobComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.blob_manager: typing.Optional[BlobManager] = None
self.blob_manager = None
@property
def component(self) -> typing.Optional[BlobManager]:
def component(self):
return self.blob_manager
async def start(self):
def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
data_store = None
datastore = None
if DHT_COMPONENT not in self.component_manager.skip_components:
dht_node: Node = self.component_manager.get_component(DHT_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
if dht_node:
data_store = dht_node.protocol.data_store
blob_dir = os.path.join(self.conf.data_dir, 'blobfiles')
if not os.path.isdir(blob_dir):
os.mkdir(blob_dir)
self.blob_manager = BlobManager(self.component_manager.loop, blob_dir, storage, self.conf, data_store)
return await self.blob_manager.setup()
datastore = dht_node._dataStore
self.blob_manager = DiskBlobManager(os.path.join(self.conf.data_dir, "blobfiles"), storage, datastore)
return self.blob_manager.setup()
async def stop(self):
self.blob_manager.stop()
def stop(self):
return self.blob_manager.stop()
async def get_status(self):
count = 0
if self.blob_manager:
count = len(self.blob_manager.completed_blob_hashes)
return {
'finished_blobs': count,
'connections': self.blob_manager.connection_manager.status
}
count = await self.blob_manager.storage.count_finished_blobs()
return {'finished_blobs': count}
class DHTComponent(Component):
@@ -316,61 +345,48 @@ class DHTComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.dht_node: Node = None
self.dht_node = None
self.upnp_component = None
self.external_udp_port = None
self.external_peer_port = None
@property
def component(self) -> typing.Optional[Node]:
def component(self):
return self.dht_node
async def get_status(self):
return {
'node_id': None if not self.dht_node else binascii.hexlify(self.dht_node.protocol.node_id),
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
'node_id': binascii.hexlify(self.component_manager.daemon.node_id),
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
}
def get_node_id(self):
node_id_filename = os.path.join(self.conf.data_dir, "node_id")
if os.path.isfile(node_id_filename):
with open(node_id_filename, "r") as node_id_file:
return base58.b58decode(str(node_id_file.read()).strip())
node_id = utils.generate_id()
with open(node_id_filename, "w") as node_id_file:
node_id_file.write(base58.b58encode(node_id).decode())
return node_id
async def start(self):
log.info("start the dht")
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.tcp_port)
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.udp_port)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
node_id = self.component_manager.daemon.node_id
if node_id is None:
node_id = generate_id()
external_ip = self.upnp_component.external_ip
if not external_ip:
log.warning("UPnP component failed to get external ip")
external_ip = await utils.get_external_ip()
external_ip = await get_external_ip()
if not external_ip:
log.warning("failed to get external ip")
self.dht_node = Node(
asyncio.get_event_loop(),
self.component_manager.peer_manager,
node_id=self.get_node_id(),
internal_udp_port=self.conf.udp_port,
udp_port=self.external_udp_port,
external_ip=external_ip,
peer_port=self.external_peer_port,
rpc_timeout=self.conf.node_rpc_timeout,
split_buckets_under_index=self.conf.split_buckets_under_index
)
self.dht_node.start(
interface=self.conf.network_interface, known_node_urls=self.conf.known_dht_nodes
node_id=node_id,
udpPort=self.conf.dht_node_port,
externalUDPPort=self.external_udp_port,
externalIP=external_ip,
peerPort=self.external_peer_port
)
await d2f(self.dht_node.start(self.conf.known_dht_nodes, block_on_join=False))
log.info("Started the dht")
async def stop(self):
self.dht_node.stop()
def stop(self):
return d2f(self.dht_node.stop())
class HashAnnouncerComponent(Component):
@@ -379,94 +395,195 @@ class HashAnnouncerComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.hash_announcer: BlobAnnouncer = None
self.hash_announcer = None
@property
def component(self) -> typing.Optional[BlobAnnouncer]:
def component(self):
return self.hash_announcer
async def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.hash_announcer = BlobAnnouncer(asyncio.get_event_loop(), dht_node, storage)
self.hash_announcer.start(self.conf.concurrent_blob_announcers)
log.info("Started blob announcer")
self.hash_announcer = DHTHashAnnouncer(self.conf, dht_node, storage)
self.hash_announcer.start()
async def stop(self):
def stop(self):
self.hash_announcer.stop()
log.info("Stopped blob announcer")
async def get_status(self):
return {
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.announce_queue)
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.hash_queue)
}
class StreamManagerComponent(Component):
component_name = STREAM_MANAGER_COMPONENT
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
class RateLimiterComponent(Component):
component_name = RATE_LIMITER_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.stream_manager: typing.Optional[StreamManager] = None
self.rate_limiter = RateLimiter()
@property
def component(self) -> typing.Optional[StreamManager]:
return self.stream_manager
async def get_status(self):
if not self.stream_manager:
return
return {
'managed_files': len(self.stream_manager.streams),
}
def component(self):
return self.rate_limiter
async def start(self):
self.rate_limiter.start()
async def stop(self):
self.rate_limiter.stop()
class PaymentRateComponent(Component):
component_name = PAYMENT_RATE_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.payment_rate_manager = OnlyFreePaymentsManager()
@property
def component(self):
return self.payment_rate_manager
async def start(self):
pass
async def stop(self):
pass
class FileManagerComponent(Component):
component_name = FILE_MANAGER_COMPONENT
depends_on = [RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
PAYMENT_RATE_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.file_manager = None
@property
def component(self):
return self.file_manager
async def get_status(self):
if not self.file_manager:
return
return {
'managed_files': len(self.file_manager.lbry_files)
}
def start(self):
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
storage = self.component_manager.get_component(DATABASE_COMPONENT)
wallet = self.component_manager.get_component(WALLET_COMPONENT)
try:
node = self.component_manager.get_component(DHT_COMPONENT)
except NameError:
node = None
log.info('Starting the file manager')
loop = asyncio.get_event_loop()
self.stream_manager = StreamManager(
loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager
)
await self.stream_manager.start()
log.info('Done setting up file manager')
async def stop(self):
self.stream_manager.stop()
sd_identifier = StreamDescriptorIdentifier()
add_lbry_file_to_sd_identifier(sd_identifier)
file_saver_factory = EncryptedFileSaverFactory(
self.conf,
self.component_manager.peer_finder,
rate_limiter,
blob_manager,
storage,
wallet,
self.conf.download_dir
)
sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
log.info('Starting the file manager')
self.file_manager = EncryptedFileManager(
self.conf, self.component_manager.peer_finder, rate_limiter,
blob_manager, wallet, payment_rate_manager, storage, sd_identifier
)
return self.file_manager.setup()
def stop(self):
return d2f(self.file_manager.stop())
class PeerProtocolServerComponent(Component):
component_name = PEER_PROTOCOL_SERVER_COMPONENT
depends_on = [UPNP_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT]
depends_on = [UPNP_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
PAYMENT_RATE_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.blob_server: typing.Optional[BlobServer] = None
self.lbry_server_port = None
@property
def component(self) -> typing.Optional[BlobServer]:
return self.blob_server
def component(self):
return self.lbry_server_port
async def start(self):
log.info("start blob server")
wallet = self.component_manager.get_component(WALLET_COMPONENT)
upnp = self.component_manager.get_component(UPNP_COMPONENT)
blob_manager: BlobManager = self.component_manager.get_component(BLOB_COMPONENT)
wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
peer_port = self.conf.tcp_port
address = await wallet.get_unused_address()
self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
self.blob_server.start_server(peer_port, interface=self.conf.network_interface)
await self.blob_server.started_listening.wait()
peer_port = self.conf.peer_port
query_handlers = {
handler.get_primary_query_identifier(): handler for handler in [
BlobRequestHandlerFactory(
self.component_manager.get_component(BLOB_COMPONENT),
wallet,
self.component_manager.get_component(PAYMENT_RATE_COMPONENT),
self.component_manager.analytics_manager
),
wallet.get_wallet_info_query_handler_factory(),
]
}
server_factory = ServerProtocolFactory(
self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers,
self.component_manager.peer_manager
)
try:
log.info("Peer protocol listening on TCP %i (ext port %i)", peer_port,
upnp.upnp_redirects.get("TCP", peer_port))
self.lbry_server_port = reactor.listenTCP(peer_port, server_factory)
except error.CannotListenError as e:
import traceback
log.error("Couldn't bind to port %d. Visit lbry.io/faq/how-to-change-port for"
" more details.", peer_port)
log.error("%s", traceback.format_exc())
raise ValueError("%s lbrynet may already be running on your computer." % str(e))
async def stop(self):
if self.blob_server:
self.blob_server.stop_server()
if self.lbry_server_port is not None:
self.lbry_server_port, old_port = None, self.lbry_server_port
log.info('Stop listening on port %s', old_port.port)
await d2f(old_port.stopListening())
class ReflectorComponent(Component):
component_name = REFLECTOR_COMPONENT
depends_on = [BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.reflector_server_port = self.conf.reflector_port
self.reflector_server = None
@property
def component(self):
return self.reflector_server
async def start(self):
log.info("Starting reflector server")
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager)
try:
self.reflector_server = await d2f(reactor.listenTCP(self.reflector_server_port, reflector_factory))
log.info('Started reflector on port %s', self.reflector_server_port)
except error.CannotListenError as e:
log.exception("Couldn't bind reflector to port %d", self.reflector_server_port)
raise ValueError(f"{e} lbrynet may already be running on your computer.")
async def stop(self):
if self.reflector_server is not None:
log.info("Stopping reflector server")
self.reflector_server, p = None, self.reflector_server
await d2f(p.stopListening())
class UPnPComponent(Component):
@@ -474,8 +591,8 @@ class UPnPComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self._int_peer_port = self.conf.tcp_port
self._int_dht_node_port = self.conf.udp_port
self._int_peer_port = self.conf.peer_port
self._int_dht_node_port = self.conf.dht_node_port
self.use_upnp = self.conf.use_upnp
self.upnp = None
self.upnp_redirects = {}
@@ -483,14 +600,14 @@ class UPnPComponent(Component):
self._maintain_redirects_task = None
@property
def component(self) -> 'UPnPComponent':
def component(self):
return self
async def _repeatedly_maintain_redirects(self, now=True):
while True:
if now:
await self._maintain_redirects()
await asyncio.sleep(360, loop=self.component_manager.loop)
await asyncio.sleep(360)
async def _maintain_redirects(self):
# setup the gateway if necessary
@@ -514,7 +631,7 @@ class UPnPComponent(Component):
if external_ip == "0.0.0.0" or not external_ip:
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
external_ip = await utils.get_external_ip()
external_ip = await get_external_ip()
if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
self.external_ip = external_ip
@@ -533,7 +650,7 @@ class UPnPComponent(Component):
self.upnp_redirects.update(upnp_redirects)
except (asyncio.TimeoutError, UPnPError):
self.upnp = None
return
return self._maintain_redirects()
elif self.upnp: # check existing redirects are still active
found = set()
mappings = await self.upnp.get_redirects()
@@ -565,7 +682,7 @@ class UPnPComponent(Component):
async def start(self):
log.info("detecting external ip")
if not self.use_upnp:
self.external_ip = await utils.get_external_ip()
self.external_ip = await get_external_ip()
return
success = False
await self._maintain_redirects()
@@ -579,10 +696,7 @@ class UPnPComponent(Component):
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
else:
log.error("failed to setup upnp")
if self.component_manager.analytics_manager:
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(
success, await self.get_status()
)
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
async def stop(self):
@@ -590,7 +704,7 @@ class UPnPComponent(Component):
await asyncio.wait([
self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
])
if self._maintain_redirects_task and not self._maintain_redirects_task.done():
if self._maintain_redirects_task is not None and not self._maintain_redirects_task.done():
self._maintain_redirects_task.cancel()
async def get_status(self):
@@ -608,11 +722,11 @@ class ExchangeRateManagerComponent(Component):
component_name = EXCHANGE_RATE_MANAGER_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
Component.__init__(self, component_manager)
self.exchange_rate_manager = ExchangeRateManager()
@property
def component(self) -> ExchangeRateManager:
def component(self):
return self.exchange_rate_manager
async def start(self):
File diff suppressed because it is too large. Load diff
+209
View File
@@ -0,0 +1,209 @@
import logging
import os
from twisted.internet import defer
from lbrynet.conf import Config
from lbrynet.schema.fee import Fee
from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
from lbrynet.p2p.Error import DownloadDataTimeout, DownloadCanceledError
from lbrynet.p2p.StreamDescriptor import download_sd_blob
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
from torba.client.constants import COIN
from lbrynet.extras.wallet.dewies import dewies_to_lbc
from lbrynet.extras.compat import f2d
INITIALIZING_CODE = 'initializing'
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
DOWNLOAD_TIMEOUT_CODE = 'timeout'
DOWNLOAD_RUNNING_CODE = 'running'
DOWNLOAD_STOPPED_CODE = 'stopped'
STREAM_STAGES = [
(INITIALIZING_CODE, 'Initializing'),
(DOWNLOAD_METADATA_CODE, 'Downloading metadata'),
(DOWNLOAD_RUNNING_CODE, 'Started stream'),
(DOWNLOAD_STOPPED_CODE, 'Paused stream'),
(DOWNLOAD_TIMEOUT_CODE, 'Stream timed out')
]
log = logging.getLogger(__name__)
class GetStream:
def __init__(self, conf: Config, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder,
rate_limiter, payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None,
timeout=None, reactor=None):
if not reactor:
from twisted.internet import reactor
self.conf = conf
self.reactor = reactor
self.timeout = timeout or conf.download_timeout
self.data_rate = data_rate or conf.data_rate
self.max_key_fee = max_key_fee or conf.max_key_fee
self.disable_max_key_fee = disable_max_key_fee or conf.disable_max_key_fee
self.download_directory = conf.download_dir
self.timeout_counter = 0
self.code = None
self.sd_hash = None
self.blob_manager = blob_manager
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.wallet = wallet
self.exchange_rate_manager = exchange_rate_manager
self.payment_rate_manager = payment_rate_manager
self.sd_identifier = sd_identifier
self.storage = storage
self.downloader = None
# fired when the download is complete
self.finished_deferred = None
# fired after the metadata and the first data blob have been downloaded
self.data_downloading_deferred = defer.Deferred(None)
self.wrote_data = False
@property
def download_path(self):
return os.path.join(self.download_directory, self.downloader.file_name)
def convert_max_fee(self):
currency, amount = self.max_key_fee['currency'], self.max_key_fee['amount']
return self.exchange_rate_manager.convert_currency(currency, "LBC", amount)
def set_status(self, status, name):
log.info("Download lbry://%s status changed to %s" % (name, status))
self.code = next(s for s in STREAM_STAGES if s[0] == status)
@defer.inlineCallbacks
def check_fee_and_convert(self, fee):
max_key_fee_amount = self.convert_max_fee()
converted_fee_amount = self.exchange_rate_manager.convert_currency(fee.currency, "LBC",
fee.amount)
if converted_fee_amount > (yield f2d(self.wallet.default_account.get_balance())):
raise InsufficientFundsError('Unable to pay the key fee of %s' % converted_fee_amount)
if converted_fee_amount > max_key_fee_amount and not self.disable_max_key_fee:
raise KeyFeeAboveMaxAllowed('Key fee {} above max allowed {}'.format(converted_fee_amount,
max_key_fee_amount))
converted_fee = {
'currency': 'LBC',
'amount': converted_fee_amount,
'address': fee.address
}
return Fee(converted_fee)
def get_downloader_factory(self, factories):
for factory in factories:
if isinstance(factory, ManagedEncryptedFileDownloaderFactory):
return factory
raise Exception(f'No suitable factory was found in {factories}')
@defer.inlineCallbacks
def get_downloader(self, factory, stream_metadata, file_name=None):
# TODO: we should use stream_metadata.options.get_downloader_options
# instead of hard-coding the options to be [self.data_rate]
downloader = yield factory.make_downloader(
stream_metadata,
self.data_rate,
self.payment_rate_manager,
self.download_directory,
file_name=file_name
)
defer.returnValue(downloader)
def _pay_key_fee(self, address, fee_lbc, name):
log.info("Pay key fee %s --> %s", dewies_to_lbc(fee_lbc), address)
reserved_points = self.wallet.reserve_points(address, fee_lbc)
if reserved_points is None:
raise InsufficientFundsError(
'Unable to pay the key fee of {} for {}'.format(dewies_to_lbc(fee_lbc), name)
)
return f2d(self.wallet.send_points_to_address(reserved_points, fee_lbc))
@defer.inlineCallbacks
def pay_key_fee(self, fee, name):
if fee is not None:
yield self._pay_key_fee(fee.address.decode(), int(fee.amount * COIN), name)
else:
defer.returnValue(None)
def finish(self, results, name):
self.set_status(DOWNLOAD_STOPPED_CODE, name)
log.info("Finished downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6],
self.download_path)
return defer.succeed(self.download_path)
def fail(self, err):
raise err
@defer.inlineCallbacks
def _initialize(self, stream_info):
# Set sd_hash and return key_fee from stream_info
self.sd_hash = stream_info.source_hash.decode()
key_fee = None
if stream_info.has_fee:
key_fee = yield self.check_fee_and_convert(stream_info.source_fee)
defer.returnValue(key_fee)
@defer.inlineCallbacks
def _create_downloader(self, sd_blob, file_name=None):
stream_metadata = yield self.sd_identifier.get_metadata_for_sd_blob(sd_blob)
factory = self.get_downloader_factory(stream_metadata.factories)
downloader = yield self.get_downloader(factory, stream_metadata, file_name)
defer.returnValue(downloader)
@defer.inlineCallbacks
def _download_sd_blob(self):
sd_blob = yield download_sd_blob(
self.conf, self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
self.wallet, self.timeout, self.conf.download_mirrors
)
defer.returnValue(sd_blob)
@defer.inlineCallbacks
def _download(self, sd_blob, name, key_fee, txid, nout, file_name=None):
self.downloader = yield self._create_downloader(sd_blob, file_name=file_name)
yield self.pay_key_fee(key_fee, name)
yield f2d(self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout)))
self.finished_deferred = self.downloader.start()
self.downloader.download_manager.progress_manager.wrote_first_data.addCallback(
self.data_downloading_deferred.callback
)
self.finished_deferred.addCallbacks(lambda result: self.finish(result, name), self.fail)
@defer.inlineCallbacks
def start(self, stream_info, name, txid, nout, file_name=None):
"""
Start download
Returns:
(tuple) Tuple containing (downloader, finished_deferred)
downloader - instance of ManagedEncryptedFileDownloader
finished_deferred - deferred callbacked when download is finished
"""
self.set_status(INITIALIZING_CODE, name)
key_fee = yield self._initialize(stream_info)
self.set_status(DOWNLOAD_METADATA_CODE, name)
try:
sd_blob = yield self._download_sd_blob()
yield self._download(sd_blob, name, key_fee, txid, nout, file_name)
self.set_status(DOWNLOAD_RUNNING_CODE, name)
log.info("Downloading lbry://%s (%s) --> %s", name, self.sd_hash[:6], self.download_path)
self.data_downloading_deferred.addTimeout(self.timeout, self.reactor)
try:
yield self.data_downloading_deferred
self.wrote_data = True
except defer.TimeoutError:
raise DownloadDataTimeout("data download timed out")
except (DownloadDataTimeout, InvalidStreamDescriptorError) as err:
raise err
defer.returnValue((self.downloader, self.finished_deferred))
def cancel(self, reason=None):
if reason:
msg = "download stream cancelled: %s" % reason
else:
msg = "download stream cancelled"
if self.data_downloading_deferred and not self.data_downloading_deferred.called:
self.data_downloading_deferred.errback(DownloadCanceledError(msg))
+92
View File
@@ -0,0 +1,92 @@
import binascii
import logging
from twisted.internet import defer, task
from lbrynet.extras.compat import f2d
from lbrynet import utils
from lbrynet.conf import Config
log = logging.getLogger(__name__)
class DHTHashAnnouncer:
def __init__(self, conf: Config, dht_node, storage, concurrent_announcers=None):
self.conf = conf
self.dht_node = dht_node
self.storage = storage
self.clock = dht_node.clock
self.peer_port = dht_node.peerPort
self.hash_queue = []
if concurrent_announcers is None:
self.concurrent_announcers = conf.concurrent_announcers
else:
self.concurrent_announcers = concurrent_announcers
self._manage_lc = None
if self.concurrent_announcers:
self._manage_lc = task.LoopingCall(self.manage)
self._manage_lc.clock = self.clock
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.concurrent_announcers or 1)
def start(self):
if self._manage_lc:
self._manage_lc.start(30)
def stop(self):
if self._manage_lc and self._manage_lc.running:
self._manage_lc.stop()
@defer.inlineCallbacks
def do_store(self, blob_hash):
storing_node_ids = yield self.dht_node.announceHaveBlob(binascii.unhexlify(blob_hash))
now = self.clock.seconds()
if storing_node_ids:
result = (now, storing_node_ids)
yield f2d(self.storage.update_last_announced_blob(blob_hash, now))
log.debug("Stored %s to %i peers", blob_hash[:16], len(storing_node_ids))
else:
result = (None, [])
self.hash_queue.remove(blob_hash)
defer.returnValue(result)
def hash_queue_size(self):
return len(self.hash_queue)
def _show_announce_progress(self, size, start):
queue_size = len(self.hash_queue)
average_blobs_per_second = float(size - queue_size) / (self.clock.seconds() - start)
log.info("Announced %i/%i blobs, %f blobs per second", size - queue_size, size, average_blobs_per_second)
@defer.inlineCallbacks
def immediate_announce(self, blob_hashes):
self.hash_queue.extend(b for b in blob_hashes if b not in self.hash_queue)
log.info("Announcing %i blobs", len(self.hash_queue))
start = self.clock.seconds()
progress_lc = task.LoopingCall(self._show_announce_progress, len(self.hash_queue), start)
progress_lc.clock = self.clock
progress_lc.start(60, now=False)
results = yield utils.DeferredDict(
{blob_hash: self.sem.run(self.do_store, blob_hash) for blob_hash in blob_hashes}
)
now = self.clock.seconds()
progress_lc.stop()
announced_to = [blob_hash for blob_hash in results if results[blob_hash][0]]
if len(announced_to) != len(results):
log.debug("Failed to announce %i blobs", len(results) - len(announced_to))
if announced_to:
log.info('Took %s seconds to announce %i of %i attempted hashes (%f hashes per second)',
now - start, len(announced_to), len(blob_hashes),
int(float(len(blob_hashes)) / float(now - start)))
defer.returnValue(results)
@defer.inlineCallbacks
def manage(self):
if not self.dht_node.contacts:
log.info("Not ready to start announcing hashes")
return
need_reannouncement = yield f2d(self.storage.get_blobs_to_announce())
if need_reannouncement:
yield self.immediate_announce(need_reannouncement)
else:
log.debug("Nothing to announce")
+70
View File
@@ -0,0 +1,70 @@
import binascii
import logging
from twisted.internet import defer
log = logging.getLogger(__name__)
class DummyPeerFinder:
"""This class finds peers which have announced to the DHT that they have certain blobs"""
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
return defer.succeed([])
class DHTPeerFinder(DummyPeerFinder):
"""This class finds peers which have announced to the DHT that they have certain blobs"""
#implements(IPeerFinder)
def __init__(self, component_manager):
"""
component_manager - an instance of ComponentManager
"""
self.component_manager = component_manager
self.peer_manager = component_manager.peer_manager
self.peers = {}
self._ongoing_searchs = {}
def find_peers_for_blob(self, blob_hash, timeout=None, filter_self=True):
"""
Find peers for blob in the DHT
blob_hash (str): blob hash to look for
timeout (int): seconds to timeout after
filter_self (bool): if True, and if a peer for a blob is itself, filter it
from the result
Returns:
list of peers for the blob
"""
if "dht" in self.component_manager.skip_components:
return defer.succeed([])
if not self.component_manager.all_components_running("dht"):
return defer.succeed([])
dht_node = self.component_manager.get_component("dht")
self.peers.setdefault(blob_hash, {(dht_node.externalIP, dht_node.peerPort,)})
if not blob_hash in self._ongoing_searchs or self._ongoing_searchs[blob_hash].called:
self._ongoing_searchs[blob_hash] = self._execute_peer_search(dht_node, blob_hash, timeout)
def _filter_self(blob_hash):
my_host, my_port = dht_node.externalIP, dht_node.peerPort
return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)}
peers = set(_filter_self(blob_hash) if filter_self else self.peers[blob_hash])
return defer.succeed([self.peer_manager.get_peer(*peer) for peer in peers])
@defer.inlineCallbacks
def _execute_peer_search(self, dht_node, blob_hash, timeout):
bin_hash = binascii.unhexlify(blob_hash)
finished_deferred = dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
timeout = timeout or self.component_manager.conf.peer_search_timeout
if timeout:
finished_deferred.addTimeout(timeout, dht_node.clock)
try:
peer_list = yield finished_deferred
self.peers[blob_hash].update({(host, port) for _, host, port in peer_list})
except defer.TimeoutError:
log.debug("DHT timed out while looking peers for blob %s after %s seconds", blob_hash, timeout)
finally:
del self._ongoing_searchs[blob_hash]
+14
View File
@@ -0,0 +1,14 @@
from lbrynet.p2p.Peer import Peer
class PeerManager:
def __init__(self):
self.peers = []
def get_peer(self, host, port):
for peer in self.peers:
if peer.host == host and peer.port == port:
return peer
peer = Peer(host, port)
self.peers.append(peer)
return peer
+77
View File
@@ -0,0 +1,77 @@
import asyncio
import logging
import os
from lbrynet.blob.EncryptedFileCreator import create_lbry_file
from lbrynet.extras.daemon.mime_types import guess_media_type
log = logging.getLogger(__name__)
def d2f(d):
return d.asFuture(asyncio.get_event_loop())
class Publisher:
def __init__(self, account, blob_manager, payment_rate_manager, storage,
lbry_file_manager, wallet, certificate):
self.account = account
self.blob_manager = blob_manager
self.payment_rate_manager = payment_rate_manager
self.storage = storage
self.lbry_file_manager = lbry_file_manager
self.wallet = wallet
self.certificate = certificate
self.lbry_file = None
async def create_and_publish_stream(self, name, bid, claim_dict, file_path, holding_address=None):
"""Create lbry file and make claim"""
log.info('Starting publish for %s', name)
if not os.path.isfile(file_path):
raise Exception(f"File {file_path} not found")
if os.path.getsize(file_path) == 0:
raise Exception(f"Cannot publish empty file {file_path}")
file_name = os.path.basename(file_path)
with open(file_path, 'rb') as read_handle:
self.lbry_file = await d2f(create_lbry_file(
self.blob_manager, self.storage, self.payment_rate_manager, self.lbry_file_manager, file_name,
read_handle
))
if 'source' not in claim_dict['stream']:
claim_dict['stream']['source'] = {}
claim_dict['stream']['source']['source'] = self.lbry_file.sd_hash
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
tx = await self.wallet.claim_name(
self.account, name, bid, claim_dict, self.certificate, holding_address
)
# check if we have a file already for this claim (if this is a publish update with a new stream)
old_stream_hashes = await self.storage.get_old_stream_hashes_for_claim_id(
tx.outputs[0].claim_id, self.lbry_file.stream_hash
)
if old_stream_hashes:
for lbry_file in filter(lambda l: l.stream_hash in old_stream_hashes,
list(self.lbry_file_manager.lbry_files)):
await d2f(self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False))
log.info("Removed old stream for claim update: %s", lbry_file.stream_hash)
await self.storage.save_content_claim(
self.lbry_file.stream_hash, tx.outputs[0].id
)
return tx
async def publish_stream(self, name, bid, claim_dict, stream_hash, holding_address=None):
"""Make a claim without creating a lbry file"""
tx = await self.wallet.claim_name(
self.account, name, bid, claim_dict, self.certificate, holding_address
)
if stream_hash: # the stream_hash returned from the db will be None if this isn't a stream we have
await self.storage.save_content_claim(
stream_hash, tx.outputs[0].id
)
self.lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == stream_hash][0]
return tx
+147 -116
View File
@@ -1,14 +1,12 @@
import asyncio
import collections
import logging
import aiohttp
import typing
from lbrynet import utils
from lbrynet.conf import Config
from lbrynet.extras import system_info
ANALYTICS_ENDPOINT = 'https://api.segment.io/v1'
ANALYTICS_TOKEN = 'Ax5LZzR1o3q3Z3WjATASDwR5rKyHH0qOIRIbLmMXn2H='
import aiohttp
from lbrynet import utils
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
from lbrynet.extras import system_info
# Things We Track
SERVER_STARTUP = 'Server Startup'
@@ -21,137 +19,103 @@ HEARTBEAT = 'Heartbeat'
CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon
NEW_CHANNEL = 'New Channel'
CREDITS_SENT = 'Credits Sent'
NEW_DOWNLOAD_STAT = 'Download'
UPNP_SETUP = "UPnP Setup"
BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded'
TIME_TO_FIRST_BYTES = "Time To First Bytes"
log = logging.getLogger(__name__)
def _event_properties(installation_id: str, session_id: str,
event_properties: typing.Optional[typing.Dict]) -> typing.Dict:
properties = {
'lbry_id': installation_id,
'session_id': session_id,
}
properties.update(event_properties or {})
return properties
def _download_properties(conf: Config, external_ip: str, resolve_duration: float,
total_duration: typing.Optional[float], download_id: str, name: str,
outpoint: str, active_peer_count: int, tried_peers_count: int,
added_fixed_peers: bool, fixed_peer_delay: float, sd_hash: str,
sd_download_duration: typing.Optional[float] = None,
head_blob_hash: typing.Optional[str] = None,
head_blob_length: typing.Optional[int] = None,
head_blob_download_duration: typing.Optional[float] = None,
error: typing.Optional[str] = None) -> typing.Dict:
return {
"external_ip": external_ip,
"download_id": download_id,
"total_duration": round(total_duration, 4),
"resolve_duration": None if not resolve_duration else round(resolve_duration, 4),
"error": error,
'name': name,
"outpoint": outpoint,
"node_rpc_timeout": conf.node_rpc_timeout,
"peer_connect_timeout": conf.peer_connect_timeout,
"blob_download_timeout": conf.blob_download_timeout,
"use_fixed_peers": len(conf.reflector_servers) > 0,
"fixed_peer_delay": fixed_peer_delay,
"added_fixed_peers": added_fixed_peers,
"active_peer_count": active_peer_count,
"tried_peers_count": tried_peers_count,
"sd_blob_hash": sd_hash,
"sd_blob_duration": None if not sd_download_duration else round(sd_download_duration, 4),
"head_blob_hash": head_blob_hash,
"head_blob_length": head_blob_length,
"head_blob_duration": None if not head_blob_download_duration else round(head_blob_download_duration, 4)
}
def _make_context(platform):
# see https://segment.com/docs/spec/common/#context
# they say they'll ignore fields outside the spec, but evidently they don't
context = {
'app': {
'version': platform['lbrynet_version'],
'build': platform['build'],
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
}
if 'desktop' in platform and 'distro' in platform:
context['os']['desktop'] = platform['desktop']
context['os']['distro'] = platform['distro']
return context
class AnalyticsManager:
class Manager:
def __init__(self, conf: Config, installation_id: str, session_id: str):
self.conf = conf
self.cookies = {}
self.url = ANALYTICS_ENDPOINT
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
self._enabled = conf.share_usage_data
self._tracked_data = collections.defaultdict(list)
self.context = _make_context(system_info.get_platform())
self.context = self._make_context(system_info.get_platform(), 'torba')
self.installation_id = installation_id
self.session_id = session_id
self.task: asyncio.Task = None
self.external_ip: typing.Optional[str] = None
@property
def is_started(self):
return self.task is not None
async def start(self):
def start(self):
if self._enabled and self.task is None:
self.external_ip = await utils.get_external_ip()
self.task = asyncio.create_task(self.run())
log.info("Start")
async def run(self):
while True:
await self._send_heartbeat()
await asyncio.sleep(1800)
self.external_ip = await utils.get_external_ip()
def stop(self):
if self.task is not None and not self.task.done():
self.task.cancel()
async def _post(self, data: typing.Dict):
async def _post(self, endpoint, data):
# there is an issue with a timing condition with keep-alive
# that is best explained here: https://github.com/mikem23/keepalive-race
#
# If you make a request, wait just the right amount of time,
# then make another request, the requests module may opt to
# reuse the connection, but by the time the server gets it the
# timeout will have expired.
#
# by forcing the connection to close, we will disable the keep-alive.
assert endpoint[0] == '/'
request_kwargs = {
'method': 'POST',
'url': self.url + '/track',
'url': self.url + endpoint,
'headers': {'Connection': 'Close'},
'auth': aiohttp.BasicAuth(self._write_key, ''),
'json': data,
'cookies': self.cookies
}
try:
async with utils.aiohttp_request(**request_kwargs) as response:
async with aiohttp.request(**request_kwargs) as response:
self.cookies.update(response.cookies)
except Exception as e:
log.debug('Encountered an exception while POSTing to %s: ', self.url + '/track', exc_info=e)
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
async def track(self, event: typing.Dict):
async def track(self, event):
"""Send a single tracking event"""
if self._enabled:
log.debug('Sending track event: %s', event)
await self._post(event)
await self._post('/track', event)
async def send_new_download_start(self, download_id, name, claim_dict):
await self._send_new_download_stats("start", download_id, name, claim_dict)
async def send_new_download_success(self, download_id, name, claim_dict):
await self._send_new_download_stats("success", download_id, name, claim_dict)
async def send_new_download_fail(self, download_id, name, claim_dict, e):
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
'message': str(e),
})
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
await self.track({
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
'event': NEW_DOWNLOAD_STAT,
'properties': self._event_properties({
'download_id': download_id,
'name': name,
'sd_hash': None if not claim_dict else claim_dict.source_hash.decode(),
'action': action,
'error': e,
}),
'context': self.context,
'timestamp': utils.isonow(),
})
async def send_upnp_setup_success_fail(self, success, status):
await self.track(
@@ -170,33 +134,20 @@ class AnalyticsManager:
async def send_server_startup_error(self, message):
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
async def send_time_to_first_bytes(self, resolve_duration: typing.Optional[float],
total_duration: typing.Optional[float], download_id: str,
name: str, outpoint: str, found_peers_count: int,
tried_peers_count: int, added_fixed_peers: bool,
fixed_peers_delay: float, sd_hash: str,
sd_download_duration: typing.Optional[float] = None,
head_blob_hash: typing.Optional[str] = None,
head_blob_length: typing.Optional[int] = None,
head_blob_duration: typing.Optional[int] = None,
error: typing.Optional[str] = None):
await self.track(self._event(TIME_TO_FIRST_BYTES, _download_properties(
self.conf, self.external_ip, resolve_duration, total_duration, download_id, name, outpoint,
found_peers_count, tried_peers_count, added_fixed_peers, fixed_peers_delay, sd_hash,
sd_download_duration, head_blob_hash, head_blob_length, head_blob_duration, error
)))
async def send_download_finished(self, download_id, name, sd_hash):
async def send_download_started(self, id_, name, claim_dict=None):
await self.track(
self._event(
DOWNLOAD_FINISHED, {
'download_id': download_id,
'name': name,
'stream_info': sd_hash
}
)
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
)
async def send_download_errored(self, err, id_, name, claim_dict, report):
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
report)
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
async def send_download_finished(self, id_, name, report, claim_dict=None):
download_properties = self._download_properties(id_, name, claim_dict, report)
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
async def send_claim_action(self, action):
await self.track(self._event(CLAIM_ACTION, {'action': action}))
@@ -209,11 +160,91 @@ class AnalyticsManager:
async def _send_heartbeat(self):
await self.track(self._event(HEARTBEAT))
def _event(self, event, properties: typing.Optional[typing.Dict] = None):
async def _update_tracked_metrics(self):
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
if should_send:
await self.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
def add_observation(self, metric, value):
self._tracked_data[metric].append(value)
def summarize_and_reset(self, metric, op=sum):
"""Apply `op` on the current values for `metric`.
This operation also resets the metric.
Returns:
a tuple (should_send, value)
"""
try:
values = self._tracked_data.pop(metric)
return True, op(values)
except KeyError:
return False, None
def _event(self, event, event_properties=None):
return {
'userId': 'lbry',
'event': event,
'properties': _event_properties(self.installation_id, self.session_id, properties),
'properties': self._event_properties(event_properties),
'context': self.context,
'timestamp': utils.isonow()
}
def _metric_event(self, metric_name, value):
return self._event(metric_name, {'value': value})
def _event_properties(self, event_properties=None):
properties = {
'lbry_id': self.installation_id,
'session_id': self.session_id,
}
properties.update(event_properties or {})
return properties
@staticmethod
def _download_properties(id_, name, claim_dict=None, report=None):
sd_hash = None if not claim_dict else claim_dict.source_hash.decode()
p = {
'download_id': id_,
'name': name,
'stream_info': sd_hash
}
if report:
p['report'] = report
return p
@staticmethod
def _download_error_properties(error, id_, name, claim_dict, report):
def error_name(err):
if not hasattr(type(err), "__name__"):
return str(type(err))
return type(err).__name__
return {
'download_id': id_,
'name': name,
'stream_info': claim_dict.source_hash.decode(),
'error': error_name(error),
'reason': str(error),
'report': report
}
@staticmethod
def _make_context(platform, wallet):
# see https://segment.com/docs/spec/common/#context
# they say they'll ignore fields outside the spec, but evidently they don't
context = {
'app': {
'version': platform['lbrynet_version'],
'build': platform['build'],
},
# TODO: expand os info to give linux/osx specific info
'os': {
'name': platform['os_system'],
'version': platform['os_release']
},
}
if 'desktop' in platform and 'distro' in platform:
context['os']['desktop'] = platform['desktop']
context['os']['distro'] = platform['distro']
return context
-6
View File
@@ -1,6 +0,0 @@
from lbrynet.conf import Config
from lbrynet.extras.cli import execute_command
def daemon_rpc(conf: Config, method: str, **kwargs):
return execute_command(conf, method, kwargs, callback=lambda data: data)
-31
View File
@@ -1,31 +0,0 @@
import logging
import aiohttp
log = logging.getLogger(__name__)
def rpc_body(method: str, rpc_id: any, **params) -> dict:
return {'jsonrpc': '2.0', 'id': rpc_id, 'method': method, 'params': {**params}}
async def jsonrpc_post(url: str, method: str, **params) -> any:
clean = params.pop('clean', True)
response = (await jsonrpc_batch(url, [rpc_body(method, 1, **params)]))[0]
if clean:
if 'error' in response:
return response['error']
return response['result']
else:
return response
async def jsonrpc_batch(url: str, calls: list, batch_size: int = 50, clean: bool = False) -> list:
headers = {'Content-Type': 'application/json'}
complete = []
batch_size = max(batch_size, 50)
for i in range(0, len(calls), batch_size):
async with aiohttp.request('POST', url, headers=headers, json=calls[i:i+batch_size]) as response:
complete += await response.json()
if clean:
complete = [body['result'] if 'result' in body else None for body in complete]
return complete
@@ -1,11 +1,10 @@
import asyncio
import aiohttp
import time
import logging
import json
from decimal import Decimal
from lbrynet.error import InvalidExchangeRateResponse, CurrencyConversionError
from lbrynet.utils import aiohttp_request
from lbrynet.p2p.Error import InvalidExchangeRateResponse, CurrencyConversionError
log = logging.getLogger(__name__)
@@ -53,8 +52,8 @@ class MarketFeed:
def is_online(self):
return self._online
async def _make_request(self) -> str:
async with aiohttp_request('get', self.url, params=self.params) as response:
async def _make_request(self):
async with aiohttp.request('get', self.url, params=self.params) as response:
return (await response.read()).decode()
def _handle_response(self, response):
@@ -220,19 +219,19 @@ class ExchangeRateManager:
def convert_currency(self, from_currency, to_currency, amount):
rates = [market.rate for market in self.market_feeds]
log.debug("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
log.info("Converting %f %s to %s, rates: %s" % (amount, from_currency, to_currency, rates))
if from_currency == to_currency:
return amount
for market in self.market_feeds:
if (market.rate_is_initialized() and market.is_online() and
market.rate.currency_pair == (from_currency, to_currency)):
return amount * Decimal(market.rate.spot)
return amount * market.rate.spot
for market in self.market_feeds:
if (market.rate_is_initialized() and market.is_online() and
market.rate.currency_pair[0] == from_currency):
return self.convert_currency(
market.rate.currency_pair[1], to_currency, amount * Decimal(market.rate.spot))
market.rate.currency_pair[1], to_currency, amount * market.rate.spot)
raise CurrencyConversionError(
f'Unable to convert {amount} from {from_currency} to {to_currency}')
+30 -149
View File
@@ -4,118 +4,25 @@ from binascii import hexlify
from datetime import datetime
from json import JSONEncoder
from ecdsa import BadSignatureError
from lbrynet.schema.claim import Claim
from lbrynet.wallet.ledger import MainNetLedger, Account
from lbrynet.wallet.transaction import Transaction, Output
from lbrynet.wallet.dewies import dewies_to_lbc
from lbrynet.stream.managed_stream import ManagedStream
from lbrynet.extras.wallet import MainNetLedger
from lbrynet.extras.wallet.transaction import Transaction, Output
from lbrynet.extras.wallet.dewies import dewies_to_lbc
log = logging.getLogger(__name__)
def encode_txo_doc():
return {
'txid': "hash of transaction in hex",
'nout': "position in the transaction",
'height': "block where transaction was recorded",
'amount': "value of the txo as a decimal",
'address': "address of who can spend the txo",
'confirmations': "number of confirmed blocks",
'is_change': "payment to change address, only available when it can be determined",
'is_mine': "payment to one of your accounts, only available when it can be determined",
'type': "one of 'claim', 'support' or 'payment'",
'name': "when type is 'claim' or 'support', this is the claim name",
'claim_id': "when type is 'claim' or 'support', this is the claim id",
'claim_op': "when type is 'claim', this determines if it is 'create' or 'update'",
'value': "when type is 'claim' or 'support' with payload, this is the decoded protobuf payload",
'value_type': "determines the type of the 'value' field: 'channel', 'stream', etc",
'protobuf': "hex encoded raw protobuf version of 'value' field",
'permanent_url': "when type is 'claim' or 'support', this is the long permanent claim URL",
'signing_channel': "for signed claims only, metadata of signing channel",
'is_channel_signature_valid': "for signed claims only, whether signature is valid",
}
def encode_tx_doc():
return {
'txid': "hash of transaction in hex",
'height': "block where transaction was recorded",
'inputs': [encode_txo_doc()],
'outputs': [encode_txo_doc()],
'total_input': "sum of inputs as a decimal",
'total_output': "sum of outputs, sans fee, as a decimal",
'total_fee': "fee amount",
'hex': "entire transaction encoded in hex",
}
def encode_account_doc():
return {
'id': 'account_id',
'is_default': 'this account is used by default',
'ledger': 'name of crypto currency and network',
'name': 'optional account name',
'seed': 'human friendly words from which account can be recreated',
'encrypted': 'if account is encrypted',
'private_key': 'extended private key',
'public_key': 'extended public key',
'address_generator': 'settings for generating addresses',
'modified_on': 'date of last modification to account settings'
}
def encode_file_doc():
return {
'streaming_url': '(str) url to stream the file using range requests',
'completed': '(bool) true if download is completed',
'file_name': '(str) name of file',
'download_directory': '(str) download directory',
'points_paid': '(float) credit paid to download file',
'stopped': '(bool) true if download is stopped',
'stream_hash': '(str) stream hash of file',
'stream_name': '(str) stream name',
'suggested_file_name': '(str) suggested file name',
'sd_hash': '(str) sd hash of file',
'download_path': '(str) download path of file',
'mime_type': '(str) mime type of file',
'key': '(str) key attached to file',
'total_bytes_lower_bound': '(int) lower bound file size in bytes',
'total_bytes': '(int) file upper bound size in bytes',
'written_bytes': '(int) written size in bytes',
'blobs_completed': '(int) number of fully downloaded blobs',
'blobs_in_stream': '(int) total blobs on stream',
'blobs_remaining': '(int) total blobs remaining to download',
'status': '(str) downloader status',
'claim_id': '(str) None if claim is not found else the claim id',
'txid': '(str) None if claim is not found else the transaction id',
'nout': '(int) None if claim is not found else the transaction output index',
'outpoint': '(str) None if claim is not found else the tx and output',
'metadata': '(dict) None if claim is not found else the claim metadata',
'channel_claim_id': '(str) None if claim is not found or not signed',
'channel_name': '(str) None if claim is not found or not signed',
'claim_name': '(str) None if claim is not found else the claim name'
}
class JSONResponseEncoder(JSONEncoder):
def __init__(self, *args, ledger: MainNetLedger, include_protobuf=False, **kwargs):
def __init__(self, *args, ledger: MainNetLedger, **kwargs):
super().__init__(*args, **kwargs)
self.ledger = ledger
self.include_protobuf = include_protobuf
def default(self, obj): # pylint: disable=method-hidden
if isinstance(obj, Account):
return self.encode_account(obj)
if isinstance(obj, ManagedStream):
return self.encode_file(obj)
if isinstance(obj, Transaction):
return self.encode_transaction(obj)
if isinstance(obj, Output):
return self.encode_output(obj)
if isinstance(obj, Claim):
return self.encode_claim(obj)
if isinstance(obj, datetime):
return obj.strftime("%Y%m%dT%H:%M:%S")
if isinstance(obj, Decimal):
@@ -136,88 +43,62 @@ class JSONResponseEncoder(JSONEncoder):
'hex': hexlify(tx.raw).decode(),
}
def encode_output(self, txo, check_signature=True):
def encode_output(self, txo):
tx_height = txo.tx_ref.height
best_height = self.ledger.headers.height
output = {
'txid': txo.tx_ref.id,
'nout': txo.position,
'height': tx_height,
'amount': dewies_to_lbc(txo.amount),
'address': txo.get_address(self.ledger),
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height
}
if txo.is_change is not None:
output['is_change'] = txo.is_change
if txo.is_my_account is not None:
output['is_mine'] = txo.is_my_account
if txo.script.is_claim_name:
output['type'] = 'claim'
output['claim_op'] = 'create'
elif txo.script.is_update_claim:
output['type'] = 'claim'
output['claim_op'] = 'update'
elif txo.script.is_support_claim:
output['type'] = 'support'
else:
output['type'] = 'payment'
if txo.script.is_claim_involved:
output.update({
'name': txo.claim_name,
'claim_id': txo.claim_id,
'permanent_url': txo.permanent_url,
})
if txo.script.is_claim_name or txo.script.is_update_claim:
output['value'] = txo.claim
output['value_type'] = txo.claim.claim_type
if self.include_protobuf:
output['protobuf'] = hexlify(txo.claim.to_bytes())
if txo.channel is not None:
output['signing_channel'] = {
'name': txo.channel.claim_name,
'claim_id': txo.channel.claim_id,
'value': txo.channel.claim
}
if check_signature and txo.claim.is_signed:
output['is_channel_signature_valid'] = False
claim = txo.claim
output['value'] = claim.claim_dict
if claim.has_signature:
output['valid_signature'] = None
if txo.channel is not None:
output['channel_name'] = txo.channel.claim_name
try:
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
output['valid_signature'] = claim.validate_signature(
txo.get_address(self.ledger), txo.channel.claim, name=txo.claim_name
)
except BadSignatureError:
pass
output['valid_signature'] = False
except ValueError:
log.exception(
'txo.id: %s, txo.channel.id:%s, output: %s',
txo.id, txo.channel.id, output
)
output['valid_signature'] = False
if txo.script.is_claim_name:
output['type'] = 'claim'
elif txo.script.is_update_claim:
output['type'] = 'update'
elif txo.script.is_support_claim:
output['type'] = 'support'
else:
output['type'] = 'basic'
return output
def encode_input(self, txi):
return self.encode_output(txi.txo_ref.txo, False) if txi.txo_ref.txo is not None else {
return self.encode_output(txi.txo_ref.txo) if txi.txo_ref.txo is not None else {
'txid': txi.txo_ref.tx_ref.id,
'nout': txi.txo_ref.position
}
def encode_account(self, account):
result = account.to_dict()
result['id'] = account.id
result.pop('certificates', None)
result['is_default'] = self.ledger.accounts[0] == account
return result
def encode_file(self, managed_stream):
file = managed_stream.as_dict()
tx_height = managed_stream.stream_claim_info.height
best_height = self.ledger.headers.height
file.update({
'height': tx_height,
'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if tx_height > 0 else None
})
return file
@staticmethod
def encode_claim(claim):
return getattr(claim, claim.claim_type).to_dict()
+4 -3
View File
@@ -1,4 +1,5 @@
import asyncio
import aiohttp
import json
import logging.handlers
import traceback
@@ -50,14 +51,14 @@ class HTTPSLogglyHandler(logging.Handler):
async def _emit(self, record):
payload = self.format(record)
async with utils.aiohttp_request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
self.cookies.update(response.cookies)
def emit(self, record):
asyncio.ensure_future(self._emit(record))
def get_loggly_handler():
handler = HTTPSLogglyHandler(LOGGLY_TOKEN)
def get_loggly_handler(loggly_token=LOGGLY_TOKEN):
handler = HTTPSLogglyHandler(loggly_token)
handler.setFormatter(JsonFormatter())
return handler
@@ -22,10 +22,6 @@ def migrate_db(conf, start, end):
from .migrate7to8 import do_migration
elif current == 8:
from .migrate8to9 import do_migration
elif current == 9:
from .migrate9to10 import do_migration
elif current == 10:
from .migrate10to11 import do_migration
else:
raise Exception("DB migration of version {} to {} is not available".format(current,
current+1))
@@ -1,54 +0,0 @@
import sqlite3
import os
import binascii
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
current_columns = []
for col_info in cursor.execute("pragma table_info('file');").fetchall():
current_columns.append(col_info[1])
if 'content_fee' in current_columns or 'saved_file' in current_columns:
connection.close()
print("already migrated")
return
cursor.execute(
"pragma foreign_keys=off;"
)
cursor.execute("""
create table if not exists new_file (
stream_hash text primary key not null references stream,
file_name text,
download_directory text,
blob_data_rate real not null,
status text not null,
saved_file integer not null,
content_fee text
);
""")
for (stream_hash, file_name, download_dir, data_rate, status) in cursor.execute("select * from file").fetchall():
saved_file = 0
if download_dir != '{stream}' and file_name != '{stream}':
try:
if os.path.isfile(os.path.join(binascii.unhexlify(download_dir).decode(),
binascii.unhexlify(file_name).decode())):
saved_file = 1
else:
download_dir, file_name = None, None
except Exception:
download_dir, file_name = None, None
else:
download_dir, file_name = None, None
cursor.execute(
"insert into new_file values (?, ?, ?, ?, ?, ?, NULL)",
(stream_hash, file_name, download_dir, data_rate, status, saved_file)
)
cursor.execute("drop table file")
cursor.execute("alter table new_file rename to file")
connection.commit()
connection.close()
@@ -3,7 +3,7 @@ import os
import json
import logging
from binascii import hexlify
from lbrynet.schema.claim import Claim
from lbrynet.schema.decode import smart_decode
log = logging.getLogger(__name__)
@@ -260,7 +260,7 @@ def do_migration(conf):
(
"%s:%i" % (claim_arg_tup[0], claim_arg_tup[1]), claim_arg_tup[2], claim_arg_tup[3],
claim_arg_tup[7], claim_arg_tup[6], claim_arg_tup[8],
Claim.from_bytes(claim_arg_tup[8]).signing_channel_id, claim_arg_tup[5], claim_arg_tup[4]
smart_decode(claim_arg_tup[8]).certificate_id, claim_arg_tup[5], claim_arg_tup[4]
)
for sd_hash, claim_arg_tup in claim_queries.items() if claim_arg_tup
] # sd_hash, (txid, nout, claim_id, name, sequence, address, height, amount, serialized)
+14 -8
View File
@@ -1,9 +1,10 @@
import sqlite3
import logging
import os
import asyncio
from lbrynet.blob.blob_info import BlobInfo
from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.p2p.Error import InvalidStreamDescriptorError
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, format_sd_info, format_blobs, validate_descriptor
from lbrynet.blob.CryptBlob import CryptBlobInfo
log = logging.getLogger(__name__)
@@ -21,13 +22,18 @@ def do_migration(conf):
"left outer join blob b ON b.blob_hash=s.blob_hash order by s.position").fetchall()
blobs_by_stream = {}
for stream_hash, position, iv, blob_hash, blob_length in blobs:
blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, blob_hash))
blobs_by_stream.setdefault(stream_hash, []).append(CryptBlobInfo(blob_hash, position, blob_length or 0, iv))
for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams:
sd = StreamDescriptor(asyncio.get_event_loop(), blob_dir, stream_name, stream_key, suggested_filename,
blobs_by_stream[stream_hash], stream_hash, sd_hash)
if sd_hash != sd.calculate_sd_hash():
log.warning("Stream for descriptor %s is invalid, cleaning it up", sd_hash)
sd_info = format_sd_info(
EncryptedFileStreamType, stream_name, stream_key,
suggested_filename, stream_hash, format_blobs(blobs_by_stream[stream_hash])
)
try:
validate_descriptor(sd_info)
except InvalidStreamDescriptorError as err:
log.warning("Stream for descriptor %s is invalid (%s), cleaning it up",
sd_hash, err.message)
blob_hashes = [blob.blob_hash for blob in blobs_by_stream[stream_hash]]
delete_stream(cursor, stream_hash, sd_hash, blob_hashes, blob_dir)
@@ -1,20 +0,0 @@
import sqlite3
import os
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
query = "select stream_hash, sd_hash from main.stream"
for stream_hash, sd_hash in cursor.execute(query).fetchall():
head_blob_hash = cursor.execute(
"select blob_hash from stream_blob where position = 0 and stream_hash = ?",
(stream_hash,)
).fetchone()
if not head_blob_hash:
continue
cursor.execute("update blob set should_announce=1 where blob_hash in (?, ?)", (sd_hash, head_blob_hash[0],))
connection.commit()
connection.close()
+158
View File
@@ -0,0 +1,158 @@
import os
types_map = {
# http://www.iana.org/assignments/media-types
'.a': 'application/octet-stream',
'.ai': 'application/postscript',
'.aif': 'audio/x-aiff',
'.aifc': 'audio/x-aiff',
'.aiff': 'audio/x-aiff',
'.au': 'audio/basic',
'.avi': 'video/x-msvideo',
'.bat': 'text/plain',
'.bcpio': 'application/x-bcpio',
'.bin': 'application/octet-stream',
'.bmp': 'image/bmp',
'.c': 'text/plain',
'.cdf': 'application/x-netcdf',
'.cpio': 'application/x-cpio',
'.csh': 'application/x-csh',
'.css': 'text/css',
'.csv': 'text/csv',
'.dll': 'application/octet-stream',
'.doc': 'application/msword',
'.dot': 'application/msword',
'.dvi': 'application/x-dvi',
'.eml': 'message/rfc822',
'.eps': 'application/postscript',
'.epub': 'application/epub+zip',
'.etx': 'text/x-setext',
'.exe': 'application/octet-stream',
'.gif': 'image/gif',
'.gtar': 'application/x-gtar',
'.h': 'text/plain',
'.hdf': 'application/x-hdf',
'.htm': 'text/html',
'.html': 'text/html',
'.ico': 'image/vnd.microsoft.icon',
'.ief': 'image/ief',
'.iges': 'model/iges',
'.jpe': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.jpg': 'image/jpeg',
'.js': 'application/javascript',
'.json': 'application/json',
'.ksh': 'text/plain',
'.latex': 'application/x-latex',
'.m1v': 'video/mpeg',
'.m3u': 'application/vnd.apple.mpegurl',
'.m3u8': 'application/vnd.apple.mpegurl',
'.man': 'application/x-troff-man',
'.markdown': 'text/markdown',
'.md': 'text/markdown',
'.me': 'application/x-troff-me',
'.mht': 'message/rfc822',
'.mhtml': 'message/rfc822',
'.mif': 'application/x-mif',
'.mov': 'video/quicktime',
'.movie': 'video/x-sgi-movie',
'.mp2': 'audio/mpeg',
'.mp3': 'audio/mpeg',
'.mp4': 'video/mp4',
'.mpa': 'video/mpeg',
'.mpe': 'video/mpeg',
'.mpeg': 'video/mpeg',
'.mpg': 'video/mpeg',
'.ms': 'application/x-troff-ms',
'.nc': 'application/x-netcdf',
'.nws': 'message/rfc822',
'.o': 'application/octet-stream',
'.obj': 'application/octet-stream',
'.oda': 'application/oda',
'.p12': 'application/x-pkcs12',
'.p7c': 'application/pkcs7-mime',
'.pbm': 'image/x-portable-bitmap',
'.pdf': 'application/pdf',
'.pfx': 'application/x-pkcs12',
'.pgm': 'image/x-portable-graymap',
'.pl': 'text/plain',
'.png': 'image/png',
'.pnm': 'image/x-portable-anymap',
'.pot': 'application/vnd.ms-powerpoint',
'.ppa': 'application/vnd.ms-powerpoint',
'.ppm': 'image/x-portable-pixmap',
'.pps': 'application/vnd.ms-powerpoint',
'.ppt': 'application/vnd.ms-powerpoint',
'.ps': 'application/postscript',
'.pwz': 'application/vnd.ms-powerpoint',
'.py': 'text/x-python',
'.pyc': 'application/x-python-code',
'.pyo': 'application/x-python-code',
'.qt': 'video/quicktime',
'.ra': 'audio/x-pn-realaudio',
'.ram': 'application/x-pn-realaudio',
'.ras': 'image/x-cmu-raster',
'.rdf': 'application/xml',
'.rgb': 'image/x-rgb',
'.roff': 'application/x-troff',
'.rtx': 'text/richtext',
'.sgm': 'text/x-sgml',
'.sgml': 'text/x-sgml',
'.sh': 'application/x-sh',
'.shar': 'application/x-shar',
'.snd': 'audio/basic',
'.so': 'application/octet-stream',
'.src': 'application/x-wais-source',
'.stl': 'model/stl',
'.sv4cpio': 'application/x-sv4cpio',
'.sv4crc': 'application/x-sv4crc',
'.svg': 'image/svg+xml',
'.swf': 'application/x-shockwave-flash',
'.t': 'application/x-troff',
'.tar': 'application/x-tar',
'.tcl': 'application/x-tcl',
'.tex': 'application/x-tex',
'.texi': 'application/x-texinfo',
'.texinfo': 'application/x-texinfo',
'.tif': 'image/tiff',
'.tiff': 'image/tiff',
'.tr': 'application/x-troff',
'.tsv': 'text/tab-separated-values',
'.txt': 'text/plain',
'.ustar': 'application/x-ustar',
'.vcf': 'text/x-vcard',
'.wav': 'audio/x-wav',
'.webm': 'video/webm',
'.wiz': 'application/msword',
'.wsdl': 'application/xml',
'.xbm': 'image/x-xbitmap',
'.xlb': 'application/vnd.ms-excel',
'.xls': 'application/vnd.ms-excel',
'.xml': 'text/xml',
'.xpdl': 'application/xml',
'.xpm': 'image/x-xpixmap',
'.xsl': 'application/xml',
'.xwd': 'image/x-xwindowdump',
'.zip': 'application/zip',
# These are non-standard types, commonly found in the wild.
'.mid': 'audio/midi',
'.midi': 'audio/midi',
'.pct': 'image/pict',
'.pic': 'image/pict',
'.pict': 'image/pict',
'.rtf': 'application/rtf',
'.xul': 'text/xul',
'.m4v': 'video/m4v',
# microsoft is special and has its own 'standard'
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
'.wmv': 'video/x-ms-wmv'
}
def guess_media_type(path):
_, ext = os.path.splitext(path)
default = f'application/x-ext-{ext}' if ext else 'application/octet-stream'
return types_map.get(ext, default)
File diff suppressed because it is too large. Load diff
+20
View File
@@ -0,0 +1,20 @@
class LoopingCallManager:
def __init__(self, calls=None):
self.calls = calls or {}
def register_looping_call(self, name, call):
assert name not in self.calls, f'{name} is already registered'
self.calls[name] = call
def start(self, name, *args):
lcall = self.calls[name]
if not lcall.running:
lcall.start(*args)
def stop(self, name):
self.calls[name].stop()
def shutdown(self):
for lcall in self.calls.values():
if lcall.running:
lcall.stop()
+65
View File
@@ -0,0 +1,65 @@
__doc__ = """
Reflector is a protocol to re-host lbry blobs and streams
Client queries and server responses follow, all dicts are encoded as json
############# Handshake request and response #############
Upon connecting, the client sends a version handshake:
{
'version': int,
}
The server replies with the same version
{
'version': int,
}
############# Stream descriptor requests and responses #############
(if sending blobs directly this is skipped)
If the client is reflecting a whole stream, they send a stream descriptor request:
{
'sd_blob_hash': str,
'sd_blob_size': int
}
The server indicates if it's aware of this stream already by requesting (or not requesting)
the stream descriptor blob. If the server has a validated copy of the sd blob, it will
include the needed_blobs field (a list of blob hashes missing from reflector) in the response.
If the server does not have the sd blob the needed_blobs field will not be included, as the
server does not know what blobs it is missing - so the client should send all of the blobs
in the stream.
{
'send_sd_blob': bool
'needed_blobs': list, conditional
}
The client may begin the file transfer of the sd blob if send_sd_blob was True.
If the client sends the blob, after receiving it the server indicates if the
transfer was successful:
{
'received_sd_blob': bool
}
If the transfer was not successful (False), the blob is added to the needed_blobs queue
############# Blob requests and responses #############
A client with blobs to reflect (either populated by the client or by the stream descriptor
response) queries if the server is ready to begin transferring a blob
{
'blob_hash': str,
'blob_size': int
}
The server replies, send_blob will be False if the server has a validated copy of the blob:
{
'send_blob': bool
}
The client may begin the raw blob file transfer if the server replied True.
If the client sends the blob, the server replies:
{
'received_blob': bool
}
If the transfer was not successful (False), the blob is re-added to the needed_blobs queue
Blob requests continue for each of the blobs the client has queued to send, when completed
the client disconnects.
"""
File renamed without changes.
+206
View File
@@ -0,0 +1,206 @@
import json
import logging
from twisted.protocols.basic import FileSender
from twisted.internet.protocol import Protocol, ClientFactory
from twisted.internet import defer, error
from lbrynet.extras.reflector.common import IncompleteResponse, REFLECTOR_V2
log = logging.getLogger(__name__)
class BlobReflectorClient(Protocol):
# Protocol stuff
def connectionMade(self):
self.blob_manager = self.factory.blob_manager
self.response_buff = b''
self.outgoing_buff = ''
self.blob_hashes_to_send = self.factory.blobs
self.next_blob_to_send = None
self.blob_read_handle = None
self.received_handshake_response = False
self.protocol_version = self.factory.protocol_version
self.file_sender = None
self.producer = None
self.streaming = False
self.reflected_blobs = []
d = self.send_handshake()
d.addErrback(
lambda err: log.warning("An error occurred immediately: %s", err.getTraceback()))
def dataReceived(self, data):
log.debug('Received %s', data)
self.response_buff += data
try:
msg = self.parse_response(self.response_buff)
except IncompleteResponse:
pass
else:
self.response_buff = b''
d = self.handle_response(msg)
d.addCallback(lambda _: self.send_next_request())
d.addErrback(self.response_failure_handler)
def connectionLost(self, reason):
if reason.check(error.ConnectionDone):
if self.reflected_blobs:
log.info('Finished sending data via reflector')
self.factory.finished_deferred.callback(self.reflected_blobs)
else:
log.info('Reflector finished: %s', reason)
self.factory.finished_deferred.callback(reason)
# IConsumer stuff
def registerProducer(self, producer, streaming):
self.producer = producer
self.streaming = streaming
if self.streaming is False:
from twisted.internet import reactor
reactor.callLater(0, self.producer.resumeProducing)
def unregisterProducer(self):
self.producer = None
def write(self, data):
self.transport.write(data)
if self.producer is not None and self.streaming is False:
from twisted.internet import reactor
reactor.callLater(0, self.producer.resumeProducing)
def send_handshake(self):
log.debug('Sending handshake')
self.write(json.dumps({'version': self.protocol_version}).encode())
return defer.succeed(None)
def parse_response(self, buff):
try:
return json.loads(buff)
except ValueError:
raise IncompleteResponse()
def response_failure_handler(self, err):
log.warning("An error occurred handling the response: %s", err.getTraceback())
def handle_response(self, response_dict):
if self.received_handshake_response is False:
return self.handle_handshake_response(response_dict)
else:
return self.handle_normal_response(response_dict)
def set_not_uploading(self):
if self.next_blob_to_send is not None:
self.read_handle.close()
self.read_handle = None
self.next_blob_to_send = None
self.file_sender = None
return defer.succeed(None)
def start_transfer(self):
assert self.read_handle is not None, \
"self.read_handle was None when trying to start the transfer"
d = self.file_sender.beginFileTransfer(self.read_handle, self)
d.addCallback(lambda _: self.read_handle.close())
return d
def handle_handshake_response(self, response_dict):
if 'version' not in response_dict:
raise ValueError("Need protocol version number!")
server_version = int(response_dict['version'])
if self.protocol_version != server_version:
raise ValueError(f"I can't handle protocol version {self.protocol_version}!")
self.received_handshake_response = True
return defer.succeed(True)
def handle_normal_response(self, response_dict):
if self.file_sender is None: # Expecting Server Info Response
if 'send_blob' not in response_dict:
raise ValueError("I don't know whether to send the blob or not!")
if response_dict['send_blob'] is True:
self.file_sender = FileSender()
return defer.succeed(True)
else:
return self.set_not_uploading()
else: # Expecting Server Blob Response
if 'received_blob' not in response_dict:
raise ValueError("I don't know if the blob made it to the intended destination!")
else:
if response_dict['received_blob']:
self.reflected_blobs.append(self.next_blob_to_send.blob_hash)
return self.set_not_uploading()
def open_blob_for_reading(self, blob):
if blob.get_is_verified():
read_handle = blob.open_for_reading()
if read_handle is not None:
log.debug('Getting ready to send %s', blob.blob_hash)
self.next_blob_to_send = blob
self.read_handle = read_handle
return None
raise ValueError(
f"Couldn't open that blob for some reason. blob_hash: {blob.blob_hash}")
def send_blob_info(self):
log.debug("Send blob info for %s", self.next_blob_to_send.blob_hash)
assert self.next_blob_to_send is not None, "need to have a next blob to send at this point"
log.debug('sending blob info')
self.write(json.dumps({
'blob_hash': self.next_blob_to_send.blob_hash,
'blob_size': self.next_blob_to_send.length
}).encode())
def disconnect(self, err):
self.transport.loseConnection()
def send_next_request(self):
if self.file_sender is not None:
# send the blob
log.debug('Sending the blob')
return self.start_transfer()
elif self.blob_hashes_to_send:
# open the next blob to send
blob_hash = self.blob_hashes_to_send[0]
log.debug('No current blob, sending the next one: %s', blob_hash)
self.blob_hashes_to_send = self.blob_hashes_to_send[1:]
blob = self.blob_manager.get_blob(blob_hash)
self.open_blob_for_reading(blob)
# send the server the next blob hash + length
return self.send_blob_info()
else:
# close connection
log.debug('No more blob hashes, closing connection')
self.transport.loseConnection()
class BlobReflectorClientFactory(ClientFactory):
protocol = BlobReflectorClient
def __init__(self, blob_manager, blobs):
self.protocol_version = REFLECTOR_V2
self.blob_manager = blob_manager
self.blobs = blobs
self.p = None
self.finished_deferred = defer.Deferred()
def buildProtocol(self, addr):
p = self.protocol()
p.factory = self
self.p = p
return p
def startFactory(self):
log.debug('Starting reflector factory')
ClientFactory.startFactory(self)
def startedConnecting(self, connector):
log.debug('Started connecting')
def clientConnectionLost(self, connector, reason):
"""If we get disconnected, reconnect to server."""
log.debug("connection lost: %s", reason.getErrorMessage())
def clientConnectionFailed(self, connector, reason):
log.debug("connection failed: %s", reason.getErrorMessage())
+346
View File
@@ -0,0 +1,346 @@
import json
import logging
from twisted.internet.error import ConnectionRefusedError
from twisted.protocols.basic import FileSender
from twisted.internet.protocol import Protocol, ClientFactory
from twisted.internet import defer, error
from lbrynet.extras.compat import f2d
from lbrynet.extras.reflector.common import IncompleteResponse, ReflectorRequestError
from lbrynet.extras.reflector.common import REFLECTOR_V1, REFLECTOR_V2
log = logging.getLogger(__name__)
class EncryptedFileReflectorClient(Protocol):
# Protocol stuff
def connectionMade(self):
log.debug("Connected to reflector")
self.response_buff = b''
self.outgoing_buff = b''
self.blob_hashes_to_send = []
self.failed_blob_hashes = []
self.next_blob_to_send = None
self.read_handle = None
self.sent_stream_info = False
self.received_descriptor_response = False
self.received_server_version = False
self.server_version = None
self.stream_descriptor = None
self.descriptor_needed = None
self.needed_blobs = []
self.reflected_blobs = []
self.file_sender = None
self.producer = None
self.streaming = False
self.blob_manager = self.factory.blob_manager
self.protocol_version = self.factory.protocol_version
self.stream_hash = self.factory.stream_hash
self.sd_hash = self.factory.sd_hash
d = self.load_descriptor()
d.addCallback(lambda _: self.send_handshake())
d.addErrback(lambda err: log.warning("An error occurred immediately: %s", err.getTraceback()))
def dataReceived(self, data):
self.response_buff += data
try:
msg = self.parse_response(self.response_buff)
except IncompleteResponse:
pass
else:
self.response_buff = b''
d = self.handle_response(msg)
d.addCallback(lambda _: self.send_next_request())
d.addErrback(self.response_failure_handler)
def store_result(self, result):
if not self.needed_blobs or len(self.reflected_blobs) == len(self.needed_blobs):
reflected = True
else:
reflected = False
d = f2d(self.blob_manager.storage.update_reflected_stream(
self.sd_hash, self.transport.getPeer().host, reflected
))
d.addCallback(lambda _: result)
return d
def connectionLost(self, reason):
# make sure blob file readers get closed
self.set_not_uploading()
if reason.check(error.ConnectionDone):
if not self.needed_blobs:
log.info("Reflector has all blobs for %s", self.stream_descriptor)
elif not self.reflected_blobs:
log.info("No more completed blobs for %s to reflect, %i are still needed",
self.stream_descriptor, len(self.needed_blobs))
else:
log.info('Finished sending reflector %i blobs for %s',
len(self.reflected_blobs), self.stream_descriptor)
result = self.reflected_blobs
elif reason.check(error.ConnectionLost):
log.warning("Stopped reflecting %s after sending %i blobs",
self.stream_descriptor, len(self.reflected_blobs))
result = self.reflected_blobs
else:
log.info('Reflector finished for %s: %s', self.stream_descriptor,
reason)
result = reason
self.factory.finished_deferred.addCallback(self.store_result)
self.factory.finished_deferred.callback(result)
# IConsumer stuff
def registerProducer(self, producer, streaming):
self.producer = producer
self.streaming = streaming
if self.streaming is False:
from twisted.internet import reactor
reactor.callLater(0, self.producer.resumeProducing)
def unregisterProducer(self):
self.producer = None
def write(self, data):
self.transport.write(data)
if self.producer is not None and self.streaming is False:
from twisted.internet import reactor
reactor.callLater(0, self.producer.resumeProducing)
def get_validated_blobs(self, blobs_in_stream):
def get_blobs(blobs):
for crypt_blob in blobs:
if crypt_blob.blob_hash and crypt_blob.length:
yield self.blob_manager.get_blob(crypt_blob.blob_hash, crypt_blob.length)
return [blob for blob in get_blobs(blobs_in_stream) if blob.get_is_verified()]
def set_blobs_to_send(self, blobs_to_send):
for blob in blobs_to_send:
if blob.blob_hash not in self.blob_hashes_to_send:
self.blob_hashes_to_send.append(blob.blob_hash)
def get_blobs_to_send(self):
def _show_missing_blobs(filtered):
if filtered:
needs_desc = "" if not self.descriptor_needed else "descriptor and "
log.info("Reflector needs %s%i blobs for stream",
needs_desc,
len(filtered))
return filtered
d = f2d(self.factory.blob_manager.storage.get_blobs_for_stream(self.stream_hash))
d.addCallback(self.get_validated_blobs)
if not self.descriptor_needed:
d.addCallback(lambda filtered:
[blob for blob in filtered if blob.blob_hash in self.needed_blobs])
d.addCallback(_show_missing_blobs)
d.addCallback(self.set_blobs_to_send)
d.addCallback(lambda _: None if self.descriptor_needed else self.set_not_uploading())
return d
def send_request(self, request_dict):
self.write(json.dumps(request_dict).encode())
def send_handshake(self):
self.send_request({'version': self.protocol_version})
@defer.inlineCallbacks
def load_descriptor(self):
if self.sd_hash:
self.stream_descriptor = yield self.factory.blob_manager.get_blob(self.sd_hash)
else:
raise ValueError("no sd hash for stream %s" % self.stream_hash)
def parse_response(self, buff):
try:
return json.loads(buff)
except ValueError:
raise IncompleteResponse()
def response_failure_handler(self, err):
log.warning("An error occurred handling the response: %s", err.getTraceback())
def handle_response(self, response_dict):
if not self.received_server_version:
return self.handle_handshake_response(response_dict)
elif not self.received_descriptor_response and self.server_version == REFLECTOR_V2:
return self.handle_descriptor_response(response_dict)
else:
return self.handle_normal_response(response_dict)
def set_not_uploading(self):
if self.next_blob_to_send is not None:
log.debug("Close %s", self.next_blob_to_send)
self.read_handle.close()
self.read_handle = None
self.next_blob_to_send = None
if self.file_sender is not None:
self.file_sender.stopProducing()
self.file_sender = None
return defer.succeed(None)
def start_transfer(self):
assert self.read_handle is not None, \
"self.read_handle was None when trying to start the transfer"
d = self.file_sender.beginFileTransfer(self.read_handle, self)
d.addCallback(lambda _: self.read_handle.close())
return d
def handle_handshake_response(self, response_dict):
if 'version' not in response_dict:
raise ValueError("Need protocol version number!")
self.server_version = int(response_dict['version'])
if self.server_version not in [REFLECTOR_V1, REFLECTOR_V2]:
raise ValueError(f"I can't handle protocol version {self.server_version}!")
self.received_server_version = True
return defer.succeed(True)
def handle_descriptor_response(self, response_dict):
if self.file_sender is None: # Expecting Server Info Response
if 'send_sd_blob' not in response_dict:
raise ReflectorRequestError("I don't know whether to send the sd blob or not!")
if response_dict['send_sd_blob'] is True:
self.file_sender = FileSender()
else:
self.received_descriptor_response = True
self.descriptor_needed = response_dict['send_sd_blob']
self.needed_blobs = response_dict.get('needed_blobs', [])
return self.get_blobs_to_send()
else: # Expecting Server Blob Response
if 'received_sd_blob' not in response_dict:
raise ValueError("I don't know if the sd blob made it to the intended destination!")
else:
self.received_descriptor_response = True
disconnect = False
if response_dict['received_sd_blob']:
self.reflected_blobs.append(self.next_blob_to_send.blob_hash)
log.info("Sent reflector descriptor %s", self.next_blob_to_send)
else:
log.warning("Reflector failed to receive descriptor %s",
self.next_blob_to_send)
disconnect = True
d = self.set_not_uploading()
if disconnect:
d.addCallback(lambda _: self.transport.loseConnection())
return d
def handle_normal_response(self, response_dict):
if self.file_sender is None: # Expecting Server Info Response
if 'send_blob' not in response_dict:
raise ValueError("I don't know whether to send the blob or not!")
if response_dict['send_blob'] is True:
self.file_sender = FileSender()
return defer.succeed(True)
else:
log.info("Reflector already has %s", self.next_blob_to_send)
return self.set_not_uploading()
else: # Expecting Server Blob Response
if 'received_blob' not in response_dict:
raise ValueError("I don't know if the blob made it to the intended destination!")
else:
if response_dict['received_blob']:
self.reflected_blobs.append(self.next_blob_to_send.blob_hash)
log.debug("Sent reflector blob %s", self.next_blob_to_send)
else:
log.warning("Reflector failed to receive blob %s", self.next_blob_to_send)
return self.set_not_uploading()
def open_blob_for_reading(self, blob):
if blob.get_is_verified():
read_handle = blob.open_for_reading()
if read_handle is not None:
log.debug('Getting ready to send %s', blob.blob_hash)
self.next_blob_to_send = blob
self.read_handle = read_handle
return defer.succeed(None)
return defer.fail(ValueError(
f"Couldn't open that blob for some reason. blob_hash: {blob.blob_hash}"))
def send_blob_info(self):
assert self.next_blob_to_send is not None, "need to have a next blob to send at this point"
r = {
'blob_hash': self.next_blob_to_send.blob_hash,
'blob_size': self.next_blob_to_send.length
}
self.send_request(r)
def send_descriptor_info(self):
assert self.stream_descriptor is not None, "need to have a sd blob to send at this point"
r = {
'sd_blob_hash': self.stream_descriptor.blob_hash,
'sd_blob_size': self.stream_descriptor.length
}
self.sent_stream_info = True
self.send_request(r)
def skip_missing_blob(self, err, blob_hash):
err.trap(ValueError)
if blob_hash not in self.failed_blob_hashes:
log.warning("Failed to reflect blob %s, reason: %s",
str(blob_hash)[:16], err.getTraceback())
self.blob_hashes_to_send.append(blob_hash)
self.failed_blob_hashes.append(blob_hash)
else:
log.warning("Failed second try reflecting blob %s, giving up, reason: %s",
str(blob_hash)[:16], err.getTraceback())
def send_next_request(self):
if self.file_sender is not None:
# send the blob
return self.start_transfer()
elif not self.sent_stream_info:
# open the sd blob to send
blob = self.stream_descriptor
d = self.open_blob_for_reading(blob)
d.addCallbacks(lambda _: self.send_descriptor_info(),
lambda err: self.skip_missing_blob(err, blob.blob_hash))
return d
elif self.blob_hashes_to_send:
# open the next blob to send
blob_hash = self.blob_hashes_to_send[0]
self.blob_hashes_to_send = self.blob_hashes_to_send[1:]
d = defer.succeed(self.blob_manager.get_blob(blob_hash))
d.addCallback(self.open_blob_for_reading)
d.addCallbacks(lambda _: self.send_blob_info(),
lambda err: self.skip_missing_blob(err, blob.blob_hash))
return d
# close connection
self.transport.loseConnection()
class EncryptedFileReflectorClientFactory(ClientFactory):
protocol = EncryptedFileReflectorClient
protocol_version = REFLECTOR_V2
def __init__(self, blob_manager, stream_hash, sd_hash):
self.blob_manager = blob_manager
self.stream_hash = stream_hash
self.sd_hash = sd_hash
self.p = None
self.finished_deferred = defer.Deferred()
def buildProtocol(self, addr):
p = self.protocol()
p.factory = self
self.p = p
return p
def startFactory(self):
log.debug('Starting reflector factory')
ClientFactory.startFactory(self)
def startedConnecting(self, connector):
log.debug('Connecting to reflector')
def clientConnectionLost(self, connector, reason):
"""If we get disconnected, reconnect to server."""
def clientConnectionFailed(self, connector, reason):
if reason.check(ConnectionRefusedError):
log.warning("Could not connect to reflector server")
else:
log.error("Reflector connection failed: %s", reason)
+27
View File
@@ -0,0 +1,27 @@
REFLECTOR_V1 = 0
REFLECTOR_V2 = 1
class ReflectorClientVersionError(Exception):
"""
Raised by reflector server if client sends an incompatible or unknown version
"""
class ReflectorRequestError(Exception):
"""
Raised by reflector server if client sends a message without the required fields
"""
class ReflectorRequestDecodeError(Exception):
"""
Raised by reflector server if client sends an invalid json request
"""
class IncompleteResponse(Exception):
"""
Raised by reflector server when client sends a portion of a json request,
used buffering the incoming request
"""
+63
View File
@@ -0,0 +1,63 @@
from twisted.internet import reactor, defer
from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory
from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory
def _is_ip(host):
try:
if len(host.split(".")) == 4 and all([0 <= int(x) <= 255 for x in host.split(".")]):
return True
return False
except ValueError:
return False
@defer.inlineCallbacks
def resolve(host):
if _is_ip(host):
ip = host
else:
ip = yield reactor.resolve(host)
defer.returnValue(ip)
@defer.inlineCallbacks
def _reflect_stream(blob_manager, stream_hash, sd_hash, reflector_server):
reflector_address, reflector_port = reflector_server[0], reflector_server[1]
factory = EncryptedFileReflectorClientFactory(blob_manager, stream_hash, sd_hash)
ip = yield resolve(reflector_address)
yield reactor.connectTCP(ip, reflector_port, factory)
result = yield factory.finished_deferred
defer.returnValue(result)
def _reflect_file(lbry_file, reflector_server):
return _reflect_stream(lbry_file.blob_manager, lbry_file.stream_hash, lbry_file.sd_hash, reflector_server)
@defer.inlineCallbacks
def _reflect_blobs(blob_manager, blob_hashes, reflector_server):
reflector_address, reflector_port = reflector_server[0], reflector_server[1]
factory = BlobReflectorClientFactory(blob_manager, blob_hashes)
ip = yield resolve(reflector_address)
yield reactor.connectTCP(ip, reflector_port, factory)
result = yield factory.finished_deferred
return result
def reflect_file(lbry_file, reflector_server):
if len(reflector_server.split(":")) == 2:
host, port = tuple(reflector_server.split(":"))
reflector_server = host, int(port)
else:
reflector_server = reflector_server, 5566
return _reflect_file(lbry_file, reflector_server)
def reflect_blob_hashes(blob_hashes, blob_manager, reflector_server):
if len(reflector_server.split(":")) == 2:
host, port = tuple(reflector_server.split(":"))
reflector_server = host, int(port)
else:
reflector_server = reflector_server, 5566
return _reflect_blobs(blob_manager, blob_hashes, reflector_server)
+325
View File
@@ -0,0 +1,325 @@
import logging
import json
from twisted.python import failure
from twisted.internet import error, defer
from twisted.internet.protocol import Protocol, ServerFactory
from lbrynet.blob.blob_file import is_valid_blobhash
from lbrynet.p2p.Error import DownloadCanceledError, InvalidBlobHashError
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorReader
from lbrynet.p2p.StreamDescriptor import save_sd_info
from lbrynet.extras.reflector.common import REFLECTOR_V1, REFLECTOR_V2
from lbrynet.extras.reflector.common import ReflectorRequestError, ReflectorClientVersionError
log = logging.getLogger(__name__)
MAXIMUM_QUERY_SIZE = 200
SEND_SD_BLOB = 'send_sd_blob'
SEND_BLOB = 'send_blob'
RECEIVED_SD_BLOB = 'received_sd_blob'
RECEIVED_BLOB = 'received_blob'
NEEDED_BLOBS = 'needed_blobs'
VERSION = 'version'
BLOB_SIZE = 'blob_size'
BLOB_HASH = 'blob_hash'
SD_BLOB_SIZE = 'sd_blob_size'
SD_BLOB_HASH = 'sd_blob_hash'
class ReflectorServer(Protocol):
def connectionMade(self):
peer_info = self.transport.getPeer()
log.debug('Connection made to %s', peer_info)
self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port)
self.blob_manager = self.factory.blob_manager
self.storage = self.factory.blob_manager.storage
self.lbry_file_manager = self.factory.lbry_file_manager
self.protocol_version = self.factory.protocol_version
self.received_handshake = False
self.peer_version = None
self.receiving_blob = False
self.incoming_blob = None
self.blob_finished_d = None
self.request_buff = b""
self.blob_writer = None
def connectionLost(self, reason=failure.Failure(error.ConnectionDone())):
log.info("Reflector upload from %s finished" % self.peer.host)
def handle_error(self, err):
log.error(err.getTraceback())
self.transport.loseConnection()
def send_response(self, response_dict):
self.transport.write(json.dumps(response_dict).encode())
############################
# Incoming blob file stuff #
############################
def clean_up_failed_upload(self, err, blob):
log.warning("Failed to receive %s", blob)
if err.check(DownloadCanceledError):
self.blob_manager.delete_blobs([blob.blob_hash])
else:
log.exception(err)
@defer.inlineCallbacks
def _on_completed_blob(self, blob, response_key):
yield self.blob_manager.blob_completed(blob, should_announce=False)
if response_key == RECEIVED_SD_BLOB:
sd_info = yield BlobStreamDescriptorReader(blob).get_info()
yield save_sd_info(self.blob_manager, blob.blob_hash, sd_info)
yield self.blob_manager.set_should_announce(blob.blob_hash, True)
else:
stream_hash = yield self.storage.get_stream_of_blob(blob.blob_hash)
if stream_hash is not None:
blob_num = yield self.storage.get_blob_num_by_hash(stream_hash,
blob.blob_hash)
if blob_num == 0:
yield self.blob_manager.set_should_announce(blob.blob_hash, True)
yield self.close_blob()
log.info("Received %s", blob)
yield self.send_response({response_key: True})
@defer.inlineCallbacks
def _on_failed_blob(self, err, response_key):
yield self.clean_up_failed_upload(err, self.incoming_blob)
yield self.send_response({response_key: False})
def handle_incoming_blob(self, response_key):
"""
Open blob for writing and send a response indicating if the transfer was
successful when finished.
response_key will either be received_blob or received_sd_blob
"""
blob = self.incoming_blob
self.blob_writer, self.blob_finished_d = blob.open_for_writing(self.peer)
self.blob_finished_d.addCallback(self._on_completed_blob, response_key)
self.blob_finished_d.addErrback(self._on_failed_blob, response_key)
def close_blob(self):
self.blob_writer.close()
self.blob_writer = None
self.blob_finished_d = None
self.incoming_blob = None
self.receiving_blob = False
####################
# Request handling #
####################
def dataReceived(self, data):
if self.receiving_blob:
self.blob_writer.write(data)
else:
log.debug('Not yet receiving blob, data needs further processing')
self.request_buff += data
msg, extra_data = self._get_valid_response(self.request_buff)
if msg is not None:
self.request_buff = b''
d = self.handle_request(msg)
d.addErrback(self.handle_error)
if self.receiving_blob and extra_data:
log.debug('Writing extra data to blob')
self.blob_writer.write(extra_data)
def _get_valid_response(self, response_msg):
extra_data = None
response = None
curr_pos = 0
while not self.receiving_blob:
next_close_paren = response_msg.find(b'}', curr_pos)
if next_close_paren != -1:
curr_pos = next_close_paren + 1
try:
response = json.loads(response_msg[:curr_pos])
except ValueError:
if curr_pos > MAXIMUM_QUERY_SIZE:
raise ValueError("Error decoding response: %s" % str(response_msg))
else:
pass
else:
extra_data = response_msg[curr_pos:]
break
else:
break
return response, extra_data
def need_handshake(self):
return self.received_handshake is False
def is_descriptor_request(self, request_dict):
if SD_BLOB_HASH not in request_dict or SD_BLOB_SIZE not in request_dict:
return False
if not is_valid_blobhash(request_dict[SD_BLOB_HASH]):
raise InvalidBlobHashError(request_dict[SD_BLOB_HASH])
return True
def is_blob_request(self, request_dict):
if BLOB_HASH not in request_dict or BLOB_SIZE not in request_dict:
return False
if not is_valid_blobhash(request_dict[BLOB_HASH]):
raise InvalidBlobHashError(request_dict[BLOB_HASH])
return True
def handle_request(self, request_dict):
if self.need_handshake():
return self.handle_handshake(request_dict)
if self.is_descriptor_request(request_dict):
return self.handle_descriptor_request(request_dict)
if self.is_blob_request(request_dict):
return self.handle_blob_request(request_dict)
raise ReflectorRequestError("Invalid request")
def handle_handshake(self, request_dict):
"""
Upon connecting, the client sends a version handshake:
{
'version': int,
}
The server replies with the same version if it is supported
{
'version': int,
}
"""
if VERSION not in request_dict:
raise ReflectorRequestError("Client should send version")
if int(request_dict[VERSION]) not in [REFLECTOR_V1, REFLECTOR_V2]:
raise ReflectorClientVersionError("Unknown version: %i" % int(request_dict[VERSION]))
self.peer_version = int(request_dict[VERSION])
log.debug('Handling handshake for client version %i', self.peer_version)
self.received_handshake = True
return self.send_handshake_response()
def send_handshake_response(self):
d = defer.succeed({VERSION: self.peer_version})
d.addCallback(self.send_response)
return d
def handle_descriptor_request(self, request_dict):
"""
If the client is reflecting a whole stream, they send a stream descriptor request:
{
'sd_blob_hash': str,
'sd_blob_size': int
}
The server indicates if it's aware of this stream already by requesting (or not requesting)
the stream descriptor blob. If the server has a validated copy of the sd blob, it will
include the needed_blobs field (a list of blob hashes missing from reflector) in the
response. If the server does not have the sd blob the needed_blobs field will not be
included, as the server does not know what blobs it is missing - so the client should send
all of the blobs in the stream.
{
'send_sd_blob': bool
'needed_blobs': list, conditional
}
The client may begin the file transfer of the sd blob if send_sd_blob was True.
If the client sends the blob, after receiving it the server indicates if the
transfer was successful:
{
'received_sd_blob': bool
}
"""
sd_blob_hash = request_dict[SD_BLOB_HASH]
sd_blob_size = request_dict[SD_BLOB_SIZE]
if self.blob_writer is None:
d = self.blob_manager.get_blob(sd_blob_hash, sd_blob_size)
d.addCallback(self.get_descriptor_response)
d.addCallback(self.send_response)
else:
self.receiving_blob = True
d = self.blob_finished_d
return d
@defer.inlineCallbacks
def get_descriptor_response(self, sd_blob):
if sd_blob.get_is_verified():
sd_info = yield BlobStreamDescriptorReader(sd_blob).get_info()
yield save_sd_info(self.blob_manager, sd_blob.blob_hash, sd_info)
yield self.storage.verify_will_announce_head_and_sd_blobs(sd_info['stream_hash'])
response = yield self.request_needed_blobs({SEND_SD_BLOB: False}, sd_info['stream_hash'])
else:
self.incoming_blob = sd_blob
self.receiving_blob = True
self.handle_incoming_blob(RECEIVED_SD_BLOB)
response = {SEND_SD_BLOB: True}
defer.returnValue(response)
@defer.inlineCallbacks
def request_needed_blobs(self, response, stream_hash):
needed_blobs = yield self.storage.get_pending_blobs_for_stream(stream_hash)
response.update({NEEDED_BLOBS: needed_blobs})
defer.returnValue(response)
def handle_blob_request(self, request_dict):
"""
A client queries if the server will accept a blob
{
'blob_hash': str,
'blob_size': int
}
The server replies, send_blob will be False if the server has a validated copy of the blob:
{
'send_blob': bool
}
The client may begin the raw blob file transfer if the server replied True.
If the client sends the blob, the server replies:
{
'received_blob': bool
}
"""
blob_hash = request_dict[BLOB_HASH]
blob_size = request_dict[BLOB_SIZE]
if self.blob_writer is None:
log.debug('Received info for blob: %s', blob_hash[:16])
d = self.blob_manager.get_blob(blob_hash, blob_size)
d.addCallback(self.get_blob_response)
d.addCallback(self.send_response)
else:
log.debug('blob is already open')
self.receiving_blob = True
d = self.blob_finished_d
return d
def get_blob_response(self, blob):
if blob.get_is_verified():
return defer.succeed({SEND_BLOB: False})
else:
self.incoming_blob = blob
self.receiving_blob = True
self.handle_incoming_blob(RECEIVED_BLOB)
d = defer.succeed({SEND_BLOB: True})
return d
class ReflectorServerFactory(ServerFactory):
protocol = ReflectorServer
def __init__(self, peer_manager, blob_manager, lbry_file_manager):
self.peer_manager = peer_manager
self.blob_manager = blob_manager
self.lbry_file_manager = lbry_file_manager
self.protocol_version = REFLECTOR_V2
def buildProtocol(self, addr):
log.debug('Creating a protocol for %s', addr)
return ServerFactory.buildProtocol(self, addr)
+9 -9
View File
@@ -2,29 +2,29 @@ import platform
import os
import logging.handlers
from lbrynet.schema import __version__ as schema_version
from lbrynet import build_type, __version__ as lbrynet_version
from torba import __version__ as torba_version
log = logging.getLogger(__name__)
def get_platform() -> dict:
os_system = platform.system()
if os.environ and 'ANDROID_ARGUMENT' in os.environ:
os_system = 'android'
p = {
"processor": platform.processor(),
"python_version": platform.python_version(),
"platform": platform.platform(),
"os_release": platform.release(),
"os_system": os_system,
"os_system": platform.system(),
"lbrynet_version": lbrynet_version,
"torba_version": torba_version,
"lbryschema_version": schema_version,
"build": build_type.BUILD, # CI server sets this during build step
}
if p["os_system"] == "Linux":
import distro
p["distro"] = distro.info()
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
try:
import distro
p["distro"] = distro.info()
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
except ModuleNotFoundError:
pass
return p
+11
View File
@@ -0,0 +1,11 @@
__node_daemon__ = 'lbrycrdd'
__node_cli__ = 'lbrycrd-cli'
__node_bin__ = ''
__node_url__ = (
'https://github.com/lbryio/lbrycrd/releases/download/v0.12.2.1/lbrycrd-linux.zip'
)
__spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest'
from lbrynet.extras.wallet.ledger import MainNetLedger, RegTestLedger
from lbrynet.extras.wallet.manager import LbryWalletManager
from lbrynet.extras.wallet.network import Network
@@ -1,51 +1,38 @@
import json
import logging
import binascii
from hashlib import sha256
from string import hexdigits
from torba.client.baseaccount import BaseAccount, HierarchicalDeterministic
from lbrynet.schema.validator import validate_claim_id
from torba.client.baseaccount import BaseAccount
from torba.client.basetransaction import TXORef
from lbrynet.schema.claim import ClaimDict
from lbrynet.schema.signer import SECP256k1, get_signer
log = logging.getLogger(__name__)
def validate_claim_id(claim_id):
if not len(claim_id) == 40:
raise Exception("Incorrect claimid length: %i" % len(claim_id))
if isinstance(claim_id, bytes):
claim_id = claim_id.decode('utf-8')
if set(claim_id).difference(hexdigits):
raise Exception("Claim id is not hex encoded")
def generate_certificate():
secp256k1_private_key = get_signer(SECP256k1).generate().private_key.to_pem()
return ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1), secp256k1_private_key
class Account(BaseAccount):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.channel_keys = {}
self.certificates = {}
@property
def hash(self) -> bytes:
h = sha256(json.dumps(self.to_dict(False)).encode())
for cert in sorted(self.channel_keys.keys()):
h.update(cert.encode())
return h.digest()
def add_certificate_private_key(self, ref: TXORef, private_key):
assert ref.id not in self.certificates, 'Trying to add a duplicate certificate.'
self.certificates[ref.id] = private_key
def apply(self, d: dict):
super().apply(d)
self.channel_keys.update(d.get('certificates', {}))
def add_channel_private_key(self, ref: TXORef, private_key):
assert ref.id not in self.channel_keys, 'Trying to add a duplicate channel private key.'
self.channel_keys[ref.id] = private_key
def get_channel_private_key(self, ref: TXORef):
return self.channel_keys.get(ref.id)
def get_certificate_private_key(self, ref: TXORef):
return self.certificates.get(ref.id)
async def maybe_migrate_certificates(self):
if not self.channel_keys:
if not self.certificates:
return
addresses = {}
@@ -59,7 +46,7 @@ class Account(BaseAccount):
}
double_hex_encoded_to_pop = []
for maybe_claim_id in list(self.channel_keys):
for maybe_claim_id in list(self.certificates):
if ':' not in maybe_claim_id:
try:
validate_claim_id(maybe_claim_id)
@@ -71,7 +58,7 @@ class Account(BaseAccount):
maybe_claim_id_bytes = maybe_claim_id_bytes.encode()
decoded_double_hex = binascii.unhexlify(maybe_claim_id_bytes).decode()
validate_claim_id(decoded_double_hex)
if decoded_double_hex in self.channel_keys:
if decoded_double_hex in self.certificates:
log.warning("don't know how to migrate certificate %s", decoded_double_hex)
else:
log.info("claim id was double hex encoded, fixing it")
@@ -80,9 +67,9 @@ class Account(BaseAccount):
continue
for double_encoded_claim_id, correct_claim_id in double_hex_encoded_to_pop:
self.channel_keys[correct_claim_id] = self.channel_keys.pop(double_encoded_claim_id)
self.certificates[correct_claim_id] = self.certificates.pop(double_encoded_claim_id)
for maybe_claim_id in list(self.channel_keys):
for maybe_claim_id in list(self.certificates):
results['total'] += 1
if ':' not in maybe_claim_id:
try:
@@ -117,8 +104,8 @@ class Account(BaseAccount):
.format(maybe_claim_id)
)
tx_nout = '{txid}:{nout}'.format(**claim)
self.channel_keys[tx_nout] = self.channel_keys[maybe_claim_id]
del self.channel_keys[maybe_claim_id]
self.certificates[tx_nout] = self.certificates[maybe_claim_id]
del self.certificates[maybe_claim_id]
log.info(
"Migrated certificate with claim_id '%s' ('%s') to a new look up key %s.",
maybe_claim_id, txo.script.values['claim_name'], tx_nout
@@ -167,11 +154,10 @@ class Account(BaseAccount):
))
async def save_max_gap(self):
if issubclass(self.address_generator, HierarchicalDeterministic):
gap = await self.get_max_gap()
self.receiving.gap = max(20, gap['max_receiving_gap'] + 1)
self.change.gap = max(6, gap['max_change_gap'] + 1)
self.wallet.save()
gap = await self.get_max_gap()
self.receiving.gap = max(20, gap['max_receiving_gap'] + 1)
self.change.gap = max(6, gap['max_change_gap'] + 1)
self.wallet.save()
def get_balance(self, confirmations=0, include_claims=False, **constraints):
if not include_claims:
@@ -179,7 +165,7 @@ class Account(BaseAccount):
return super().get_balance(confirmations, **constraints)
@classmethod
def get_private_key_from_seed(cls, ledger, seed: str, password: str):
def get_private_key_from_seed(cls, ledger: 'baseledger.BaseLedger', seed: str, password: str):
return super().get_private_key_from_seed(
ledger, seed, password or 'lbryum'
)
@@ -187,30 +173,35 @@ class Account(BaseAccount):
@classmethod
def from_dict(cls, ledger, wallet, d: dict) -> 'Account':
account = super().from_dict(ledger, wallet, d)
account.channel_keys = d.get('certificates', {})
account.certificates = d.get('certificates', {})
return account
def to_dict(self, include_channel_keys=True):
def to_dict(self):
d = super().to_dict()
if include_channel_keys:
d['certificates'] = self.channel_keys
d['certificates'] = self.certificates
return d
async def get_details(self, **kwargs):
details = await super().get_details(**kwargs)
details['certificates'] = len(self.channel_keys)
details['certificates'] = len(self.certificates)
return details
def get_claim(self, claim_id=None, txid=None, nout=None):
if claim_id is not None:
return self.ledger.db.get_claims(account=self, claim_id=claim_id)
elif txid is not None and nout is not None:
return self.ledger.db.get_claims(**{'account': self, 'txo.txid': txid, 'txo.position': nout})
@staticmethod
def constraint_spending_utxos(constraints):
def constraint_utxos_sans_claims(constraints):
constraints.update({'is_claim': 0, 'is_update': 0, 'is_support': 0})
def get_utxos(self, **constraints):
self.constraint_spending_utxos(constraints)
self.constraint_utxos_sans_claims(constraints)
return super().get_utxos(**constraints)
def get_utxo_count(self, **constraints):
self.constraint_spending_utxos(constraints)
self.constraint_utxos_sans_claims(constraints)
return super().get_utxo_count(**constraints)
def get_claims(self, **constraints):
@@ -219,23 +210,32 @@ class Account(BaseAccount):
def get_claim_count(self, **constraints):
return self.ledger.db.get_claim_count(account=self, **constraints)
def get_streams(self, **constraints):
return self.ledger.db.get_streams(account=self, **constraints)
def get_stream_count(self, **constraints):
return self.ledger.db.get_stream_count(account=self, **constraints)
def get_channels(self, **constraints):
return self.ledger.db.get_channels(account=self, **constraints)
def get_channel_count(self, **constraints):
return self.ledger.db.get_channel_count(account=self, **constraints)
def get_supports(self, **constraints):
return self.ledger.db.get_supports(account=self, **constraints)
async def send_to_addresses(self, amount, addresses, broadcast=False):
tx_class = self.ledger.transaction_class
tx = await tx_class.create(
inputs=[],
outputs=[
tx_class.output_class.pay_pubkey_hash(amount, self.ledger.address_to_hash160(address))
for address in addresses
],
funding_accounts=[self],
change_account=self
)
def get_support_count(self, **constraints):
return self.ledger.db.get_support_count(account=self, **constraints)
if broadcast:
await self.ledger.broadcast(tx)
else:
await self.ledger.release_outputs(
[txi.txo_ref.txo for txi in tx.inputs]
)
return tx
async def release_all_outputs(self):
await self.ledger.db.release_all_outputs(self)
@@ -74,10 +74,9 @@ def verify_proof(proof, root_hash, name):
if 'txhash' in proof and 'nOut' in proof:
if not verified_value:
raise InvalidProofError("mismatch between proof claim and outcome")
target = reverse_computed_name[::-1].encode('ISO-8859-1').decode()
if 'txhash' in proof and 'nOut' in proof:
if name != target:
if name != reverse_computed_name[::-1]:
raise InvalidProofError("name did not match proof")
if not name.startswith(target):
if not name.startswith(reverse_computed_name[::-1]):
raise InvalidProofError("name fragment does not match proof")
return True
Loaded 100 of 303 files, more files were not shown because too many files have changed in this diff. Show more