Compare commits
129
Commits
v0.30.2
...
release_time
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
574c2ce794 | ||
|
|
7f5541eb5c | ||
|
|
7b49241c38 | ||
|
|
da716b2e3d | ||
|
|
a144fae8b2 | ||
|
|
0d90b82287 | ||
|
|
567c23e10f | ||
|
|
16970b7d37 | ||
|
|
8593cd362a | ||
|
|
2dbf96da81 | ||
|
|
83c7797187 | ||
|
|
6606deafa5 | ||
|
|
19b0d59159 | ||
|
|
9ccb3fa2a3 | ||
|
|
48533df523 | ||
|
|
1ae1137005 | ||
|
|
d5f2832b2f | ||
|
|
d0230b4893 | ||
|
|
a6cd53b97c | ||
|
|
7a28171a72 | ||
|
|
150bcb1116 | ||
|
|
6fe9c5bf00 | ||
|
|
3a86fc6680 | ||
|
|
2805cbb331 | ||
|
|
68d7a7014c | ||
|
|
883fa267a5 | ||
|
|
5f2c3ac965 | ||
|
|
bf8e32ebd2 | ||
|
|
5971be5bef | ||
|
|
677d76a3e4 | ||
|
|
6ed3207cb7 | ||
|
|
be6448b74f | ||
|
|
b707ee4844 | ||
|
|
0e9888be3b | ||
|
|
68b56b7858 | ||
|
|
d32697f611 | ||
|
|
3c54b2bcd7 | ||
|
|
899b182811 | ||
|
|
0dfb5e008d | ||
|
|
abf94357eb | ||
|
|
36aded3830 | ||
|
|
d7ebf50602 | ||
|
|
523c4c86a9 | ||
|
|
74cccdbfc7 | ||
|
|
70471eebfa | ||
|
|
6339224980 | ||
|
|
fe97f7a119 | ||
|
|
f8ed605da2 | ||
|
|
e38ef2af1f | ||
|
|
5d8e627d02 | ||
|
|
c9d4ad5450 | ||
|
|
31586bf588 | ||
|
|
974d3c83e3 | ||
|
|
8dc8ca382b | ||
|
|
1dbb250413 | ||
|
|
9f9ec692c7 | ||
|
|
2c1b54c76c | ||
|
|
7314494fb9 | ||
|
|
e74a5e8ef3 | ||
|
|
b9af999739 | ||
|
|
2de155f90f | ||
|
|
ab3db2e626 | ||
|
|
f9d26b04d8 | ||
|
|
24e0673198 | ||
|
|
833261801f | ||
|
|
6c8cbc1b9e | ||
|
|
6ae4e68d1c | ||
|
|
d22a2dee33 | ||
|
|
0f1ed589a5 | ||
|
|
b032e99dd1 | ||
|
|
fd7811f38d | ||
|
|
071db3a73d | ||
|
|
950c122850 | ||
|
|
e0386ea9dc | ||
|
|
6c3eee0f0e | ||
|
|
4c23add193 | ||
|
|
248d1a18c7 | ||
|
|
17d1b85423 | ||
|
|
b0deb63129 | ||
|
|
ebdb33bd11 | ||
|
|
69ccbda168 | ||
|
|
a38643cca6 | ||
|
|
712b19605e | ||
|
|
8a5f3842fc | ||
|
|
db3e578bcd | ||
|
|
7e22adff7c | ||
|
|
76fbaac658 | ||
|
|
c4fac29ecb | ||
|
|
33392b3351 | ||
|
|
0613cf2d59 | ||
|
|
a930ca0631 | ||
|
|
63a7271008 | ||
|
|
5ca0ebcbb2 | ||
|
|
2c84b7a7dd | ||
|
|
71a0c704bd | ||
|
|
d30e7cd29e | ||
|
|
aeb7c6fe76 | ||
|
|
134c20da93 | ||
|
|
893b4fc3ca | ||
|
|
9b27e81940 | ||
|
|
72c0acf269 | ||
|
|
7be53ad692 | ||
|
|
16e596ec87 | ||
|
|
3fdcb80960 | ||
|
|
26a3221bac | ||
|
|
eb16cb0aa8 | ||
|
|
b5340505e5 | ||
|
|
4bb8129bb7 | ||
|
|
7f838d3a31 | ||
|
|
c186c31bf7 | ||
|
|
86fb99a37c | ||
|
|
6c52450858 | ||
|
|
fb4e94b04a | ||
|
|
248baf58b4 | ||
|
|
0845d65f4e | ||
|
|
dcc5415648 | ||
|
|
3c38a88b22 | ||
|
|
187b7670b1 | ||
|
|
049c17ff08 | ||
|
|
8de97bb441 | ||
|
|
97c0092923 | ||
|
|
e65e1fb5a2 | ||
|
|
20dd1e55f5 | ||
|
|
6aa709e34e | ||
|
|
e73f121c08 | ||
|
|
3b9d6c4f8e | ||
|
|
69b60c19c3 | ||
|
|
04278cb1ab | ||
|
|
d1da42bea8 |
No files matched your search
@@ -0,0 +1,7 @@
|
||||
.git
|
||||
.tox
|
||||
__pycache__
|
||||
dist
|
||||
lbrynet.egg-info
|
||||
docs
|
||||
tests
|
||||
+7
-5
@@ -9,8 +9,8 @@ jobs:
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid==2.0.4
|
||||
# newer astroid fails in pylint so we pre-install older version
|
||||
- pip install astroid==2.0.4 aiohttp==3.4.4
|
||||
# newer astroid and aiohttp fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
@@ -24,8 +24,9 @@ jobs:
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .[test]
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -v tests.unit.wallet
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics tests.unit.components tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.test_cli tests.unit.test_customLogger tests.unit.schema
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.schema tests.unit.wallet tests.unit.components tests.unit.test_conf
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.test_cli
|
||||
#- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
@@ -110,7 +111,8 @@ jobs:
|
||||
env: OS=mac
|
||||
cache: false
|
||||
before_install:
|
||||
- brew upgrade python
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
# <img src="https://raw.githubusercontent.com/lbryio/lbry/master/lbry.png" alt="LBRY" width="48" height="36" /> LBRY SDK [](https://travis-ci.org/lbryio/lbry) [](https://codecov.io/gh/lbryio/lbry)
|
||||
|
||||
LBRY is an open-source protocol providing distribution, discovery, and purchase of digital content (data) via a decentralized peer-to-peer network. It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
LBRY is a decentralized peer-to-peer network providing distribution, discovery, and purchase of digital content (data). It utilizes the [LBRY blockchain](https://github.com/lbryio/lbrycrd) as a global namespace and database of digital content. Blockchain entries contain searchable content metadata, identities, rights and access rules. LBRY also provides a data network that consists of peers (seeders) uploading and downloading data from other peers, possibly in exchange for payments, as well as a distributed hash table used by peers to discover other peers.
|
||||
|
||||
This project provides an SDK for building applications using the LBRY protocol. The SDK is written in Python 3.7+ using Twisted. For other languages or platforms, a JSON-RPC API is provided.
|
||||
LBRY SDK for Python is currently the most full featured implementation of the LBRY Network protocols and includes many useful components and tools for building decentralized applications. Primary features and components:
|
||||
|
||||
* Built on Python 3.7+ and `asyncio`.
|
||||
* DHT (Distributed Hash Table) implementation for finding peers ([lbrynet.dht](https://github.com/lbryio/lbry/tree/master/lbrynet/dht)).
|
||||
* Blob exchange protocol for downloading content and negotiating payments ([lbrynet.blob_exchange](https://github.com/lbryio/lbry/tree/master/lbrynet/blob_exchange)).
|
||||
* Protobuf schema for encoding and decoding metadata stored on the blockchain ([lbrynet.schema](https://github.com/lbryio/lbry/tree/master/lbrynet/schema)).
|
||||
* Wallet implementation for the LBRY blockchain ([lbrynet.extras.wallet](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/wallet)).
|
||||
* Daemon with a JSON-RPC API to ease building end user applications in any language and for automating various tasks ([lbrynet.extras.daemon](https://github.com/lbryio/lbry/tree/master/lbrynet/extras/daemon)).
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -38,7 +45,7 @@ We take security seriously. Please contact security@lbry.io regarding any securi
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@jackrobison](mailto:jack@lbry.io).
|
||||
The primary contact for this project is [@eukreign](mailto:lex@lbry.io).
|
||||
|
||||
## Additional information and links
|
||||
|
||||
|
||||
@@ -17,9 +17,6 @@ dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
|
||||
use_auth_http: True
|
||||
use_https: True
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
|
||||
+1
-7
@@ -1,10 +1,4 @@
|
||||
import logging
|
||||
from lbrynet.custom_logger import install_logger
|
||||
|
||||
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.30.2"
|
||||
__version__ = "0.30.5"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
install_logger()
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -9,6 +9,7 @@ from binascii import hexlify
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
|
||||
from lbrynet.blob.CryptStreamCreator import CryptStreamCreator
|
||||
@@ -119,13 +120,13 @@ def create_lbry_file(blob_manager, storage, payment_rate_manager, lbry_file_mana
|
||||
sd_hash = yield descriptor_writer.create_descriptor(sd_info)
|
||||
|
||||
log.debug("saving the stream")
|
||||
yield storage.store_stream(
|
||||
yield f2d(storage.store_stream(
|
||||
sd_info['stream_hash'], sd_hash, sd_info['stream_name'], sd_info['key'],
|
||||
sd_info['suggested_file_name'], sd_info['blobs']
|
||||
)
|
||||
))
|
||||
log.debug("adding to the file manager")
|
||||
lbry_file = yield lbry_file_manager.add_published_file(
|
||||
lbry_file = yield f2d(lbry_file_manager.add_published_file(
|
||||
sd_info['stream_hash'], sd_hash, hexlify(file_directory.encode()), payment_rate_manager,
|
||||
payment_rate_manager.min_blob_data_payment_rate
|
||||
)
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
@@ -5,12 +5,12 @@ import logging
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from lbrynet import conf
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
|
||||
from lbrynet.utils import short_hash
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
|
||||
@@ -34,12 +34,12 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage, lbry_file_manager,
|
||||
payment_rate_manager, wallet, download_directory, file_name, stream_name, sd_hash, key,
|
||||
suggested_file_name, download_mirrors=None):
|
||||
def __init__(self, conf: Config, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
|
||||
lbry_file_manager, payment_rate_manager, wallet, download_directory, file_name, stream_name,
|
||||
sd_hash, key, suggested_file_name, download_mirrors=None):
|
||||
super().__init__(
|
||||
stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name
|
||||
conf, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager,
|
||||
wallet, download_directory, key, stream_name, file_name
|
||||
)
|
||||
self.sd_hash = sd_hash
|
||||
self.rowid = rowid
|
||||
@@ -55,9 +55,9 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.channel_name = None
|
||||
self.metadata = None
|
||||
self.mirror = None
|
||||
if download_mirrors or conf.settings['download_mirrors']:
|
||||
if download_mirrors or conf.download_mirrors:
|
||||
self.mirror = HTTPBlobDownloader(
|
||||
self.blob_manager, servers=download_mirrors or conf.settings['download_mirrors']
|
||||
self.blob_manager, servers=download_mirrors or conf.download_mirrors
|
||||
)
|
||||
|
||||
def set_claim_info(self, claim_info):
|
||||
@@ -70,13 +70,11 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.channel_name = claim_info['channel_name']
|
||||
self.metadata = claim_info['value']['stream']['metadata']
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claim_info(self, include_supports=True):
|
||||
claim_info = yield self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
async def get_claim_info(self, include_supports=True):
|
||||
claim_info = await self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
if claim_info:
|
||||
self.set_claim_info(claim_info)
|
||||
|
||||
defer.returnValue(claim_info)
|
||||
return claim_info
|
||||
|
||||
@property
|
||||
def saving_status(self):
|
||||
@@ -101,16 +99,15 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
if self.mirror:
|
||||
self.mirror.stop()
|
||||
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
yield EncryptedFileDownloader.stop(self, err=err)
|
||||
yield super().stop(err)
|
||||
if change_status is True:
|
||||
status = yield self._save_status()
|
||||
defer.returnValue(status)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def status(self):
|
||||
blobs = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
async def status(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
|
||||
completed_blobs = yield self.blob_manager.completed_blobs(blob_hashes)
|
||||
completed_blobs = self.blob_manager.completed_blobs(blob_hashes)
|
||||
num_blobs_completed = len(completed_blobs)
|
||||
num_blobs_known = len(blob_hashes)
|
||||
|
||||
@@ -120,9 +117,10 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
status = "stopped"
|
||||
else:
|
||||
status = "running"
|
||||
defer.returnValue(EncryptedFileStatusReport(
|
||||
|
||||
return EncryptedFileStatusReport(
|
||||
self.file_name, num_blobs_completed, num_blobs_known, status
|
||||
))
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start(self):
|
||||
@@ -150,7 +148,7 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_RUNNING
|
||||
status = yield self.lbry_file_manager.change_lbry_file_status(self, status)
|
||||
self._saving_status = False
|
||||
defer.returnValue(status)
|
||||
return status
|
||||
|
||||
def save_status(self):
|
||||
return self._save_status()
|
||||
@@ -180,10 +178,10 @@ class ManagedEncryptedFileDownloaderFactory:
|
||||
if file_name:
|
||||
file_name = hexlify(file_name.encode())
|
||||
hex_download_directory = hexlify(download_directory.encode())
|
||||
lbry_file = yield self.lbry_file_manager.add_downloaded_file(
|
||||
lbry_file = yield f2d(self.lbry_file_manager.add_downloaded_file(
|
||||
stream_hash, metadata.source_blob_hash, hex_download_directory, payment_rate_manager,
|
||||
data_rate, file_name=file_name, download_mirrors=download_mirrors
|
||||
)
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -3,11 +3,13 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
import random
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet import conf
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.reflector.reupload import reflect_file
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
@@ -27,9 +29,11 @@ class EncryptedFileManager:
|
||||
# when reflecting files, reflect up to this many files at a time
|
||||
CONCURRENT_REFLECTS = 5
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, wallet, payment_rate_manager, storage, sd_identifier):
|
||||
self.auto_re_reflect = conf.settings['reflect_uploads'] and conf.settings['auto_re_reflect_interval'] > 0
|
||||
self.auto_re_reflect_interval = conf.settings['auto_re_reflect_interval']
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, wallet,
|
||||
payment_rate_manager, storage, sd_identifier):
|
||||
self.conf = conf
|
||||
self.auto_re_reflect = conf.reflect_uploads and conf.auto_re_reflect_interval > 0
|
||||
self.auto_re_reflect_interval = conf.auto_re_reflect_interval
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
@@ -41,11 +45,9 @@ class EncryptedFileManager:
|
||||
self.lbry_files = []
|
||||
self.lbry_file_reflector = task.LoopingCall(self.reflect_lbry_files)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
yield self._add_to_sd_identifier()
|
||||
yield self._start_lbry_files()
|
||||
log.info("Started file manager")
|
||||
self._add_to_sd_identifier()
|
||||
return self._start_lbry_files()
|
||||
|
||||
def get_lbry_file_status(self, lbry_file):
|
||||
return self.storage.get_lbry_file_status(lbry_file.rowid)
|
||||
@@ -55,7 +57,7 @@ class EncryptedFileManager:
|
||||
|
||||
def change_lbry_file_status(self, lbry_file, status):
|
||||
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
|
||||
return self.storage.change_file_status(lbry_file.rowid, status)
|
||||
return f2d(self.storage.change_file_status(lbry_file.rowid, status))
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
@@ -79,6 +81,7 @@ class EncryptedFileManager:
|
||||
def _get_lbry_file(self, rowid, stream_hash, payment_rate_manager, sd_hash, key,
|
||||
stream_name, file_name, download_directory, suggested_file_name, download_mirrors=None):
|
||||
return ManagedEncryptedFileDownloader(
|
||||
self.conf,
|
||||
rowid,
|
||||
stream_hash,
|
||||
self.peer_finder,
|
||||
@@ -115,10 +118,9 @@ class EncryptedFileManager:
|
||||
except Exception:
|
||||
log.warning("Failed to start %i", file_info.get('rowid'))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start_lbry_files(self):
|
||||
files = yield self.storage.get_all_lbry_files()
|
||||
claim_infos = yield self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
async def _start_lbry_files(self):
|
||||
files = await self.storage.get_all_lbry_files()
|
||||
claim_infos = await self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
prm = self.payment_rate_manager
|
||||
|
||||
log.info("Starting %i files", len(files))
|
||||
@@ -151,14 +153,13 @@ class EncryptedFileManager:
|
||||
log.info("Stopping %i lbry files", len(self.lbry_files))
|
||||
yield defer.DeferredList([self._stop_lbry_file(lbry_file) for lbry_file in list(self.lbry_files)])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
async def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = stream_metadata['suggested_file_name']
|
||||
rowid = yield self.storage.save_published_file(
|
||||
rowid = await self.storage.save_published_file(
|
||||
stream_hash, file_name, download_directory, blob_data_rate, status
|
||||
)
|
||||
lbry_file = self._get_lbry_file(
|
||||
@@ -166,37 +167,36 @@ class EncryptedFileManager:
|
||||
stream_metadata['suggested_file_name'], download_mirrors=None
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info()
|
||||
await lbry_file.get_claim_info()
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
async def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
blob_data_rate=None, status=None, file_name=None, download_mirrors=None):
|
||||
status = status or ManagedEncryptedFileDownloader.STATUS_STOPPED
|
||||
payment_rate_manager = payment_rate_manager or self.payment_rate_manager
|
||||
blob_data_rate = blob_data_rate or payment_rate_manager.min_blob_data_payment_rate
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = file_name or stream_metadata['suggested_file_name']
|
||||
|
||||
# when we save the file we'll atomic touch the nearest file to the suggested file name
|
||||
# that doesn't yet exist in the download directory
|
||||
rowid = yield self.storage.save_downloaded_file(
|
||||
rowid = await self.storage.save_downloaded_file(
|
||||
stream_hash, hexlify(os.path.basename(unhexlify(file_name))), download_directory, blob_data_rate
|
||||
)
|
||||
file_name = (yield self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
file_name = (await self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
lbry_file = self._get_lbry_file(
|
||||
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
|
||||
stream_metadata['suggested_file_name'], download_mirrors
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info(include_supports=False)
|
||||
await lbry_file.get_claim_info(include_supports=False)
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_lbry_file(self, lbry_file, delete_file=False):
|
||||
@@ -222,7 +222,7 @@ class EncryptedFileManager:
|
||||
del self.storage.content_claim_callbacks[lbry_file.stream_hash]
|
||||
|
||||
yield lbry_file.delete_data()
|
||||
yield self.storage.delete_stream(lbry_file.stream_hash)
|
||||
yield f2d(self.storage.delete_stream(lbry_file.stream_hash))
|
||||
|
||||
if delete_file and os.path.isfile(full_path):
|
||||
os.remove(full_path)
|
||||
@@ -240,10 +240,10 @@ class EncryptedFileManager:
|
||||
def reflect_lbry_files(self):
|
||||
sem = defer.DeferredSemaphore(self.CONCURRENT_REFLECTS)
|
||||
ds = []
|
||||
sd_hashes_to_reflect = yield self.storage.get_streams_to_re_reflect()
|
||||
sd_hashes_to_reflect = yield f2d(self.storage.get_streams_to_re_reflect())
|
||||
for lbry_file in self.lbry_files:
|
||||
if lbry_file.sd_hash in sd_hashes_to_reflect:
|
||||
ds.append(sem.run(reflect_file, lbry_file))
|
||||
ds.append(sem.run(reflect_file, lbry_file, random.choice(self.conf.reflector_servers)))
|
||||
yield defer.DeferredList(ds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import logging
|
||||
from twisted.internet import defer, threads
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
@@ -111,23 +111,13 @@ class BlobFile:
|
||||
if not self.writers and not self.readers:
|
||||
self._verified = False
|
||||
self.saved_verified_blob = False
|
||||
|
||||
def delete_from_file_system():
|
||||
try:
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
log.warning("An error occurred deleting %s: %s",
|
||||
str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
except Exception as e:
|
||||
log.exception("An error occurred deleting %s:", str(self.file_path), exc_info=e)
|
||||
else:
|
||||
return defer.fail(Failure(
|
||||
ValueError("File is currently being read or written and cannot be deleted")))
|
||||
raise ValueError("File is currently being read or written and cannot be deleted")
|
||||
|
||||
@property
|
||||
def verified(self):
|
||||
|
||||
@@ -3,6 +3,7 @@ from binascii import unhexlify
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.p2p.client.BlobRequester import BlobRequester
|
||||
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.p2p.client.DownloadManager import DownloadManager
|
||||
@@ -37,7 +38,7 @@ class CryptStreamDownloader:
|
||||
|
||||
#implements(IStreamDownloader)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
|
||||
key, stream_name):
|
||||
"""Initialize a CryptStreamDownloader
|
||||
|
||||
@@ -55,7 +56,7 @@ class CryptStreamDownloader:
|
||||
@return:
|
||||
|
||||
"""
|
||||
|
||||
self.conf = conf
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
@@ -4,6 +4,8 @@ import traceback
|
||||
from binascii import hexlify, unhexlify
|
||||
from twisted.internet import defer, threads
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
@@ -17,9 +19,9 @@ log = logging.getLogger(__name__)
|
||||
class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
"""Classes which inherit from this class download LBRY files"""
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
storage, payment_rate_manager, wallet, key, stream_name, file_name):
|
||||
super().__init__(peer_finder, rate_limiter, blob_manager,
|
||||
super().__init__(conf, peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, key, stream_name)
|
||||
self.stream_hash = stream_hash
|
||||
self.storage = storage
|
||||
@@ -28,16 +30,15 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_data(self):
|
||||
crypt_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
crypt_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
blob_hashes = [b.blob_hash for b in crypt_infos if b.blob_hash]
|
||||
sd_hash = yield self.storage.get_sd_blob_hash_for_stream(self.stream_hash)
|
||||
sd_hash = yield f2d(self.storage.get_sd_blob_hash_for_stream(self.stream_hash))
|
||||
blob_hashes.append(sd_hash)
|
||||
yield self.blob_manager.delete_blobs(blob_hashes)
|
||||
|
||||
def stop(self, err=None):
|
||||
d = self._close_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
|
||||
return d
|
||||
self._close_output()
|
||||
return super().stop(err=err)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
@@ -65,14 +66,9 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
def _close_output(self):
|
||||
pass
|
||||
|
||||
def get_total_bytes(self):
|
||||
d = self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def calculate_size(blobs):
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
d.addCallback(calculate_size)
|
||||
return d
|
||||
async def get_total_bytes(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
def get_total_bytes_cached(self):
|
||||
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
|
||||
@@ -102,7 +98,8 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
class EncryptedFileDownloaderFactory:
|
||||
#implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet):
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet):
|
||||
self.conf = conf
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
@@ -135,9 +132,9 @@ class EncryptedFileDownloaderFactory:
|
||||
|
||||
|
||||
class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name):
|
||||
super().__init__(stream_hash, peer_finder, rate_limiter,
|
||||
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
|
||||
payment_rate_manager, wallet, download_directory, key, stream_name, file_name):
|
||||
super().__init__(conf, stream_hash, peer_finder, rate_limiter,
|
||||
blob_manager, storage, payment_rate_manager,
|
||||
wallet, key, stream_name, file_name)
|
||||
self.download_directory = unhexlify(download_directory).decode()
|
||||
@@ -147,10 +144,6 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __str__(self):
|
||||
return str(self.file_written_to)
|
||||
|
||||
def stop(self, err=None):
|
||||
d = EncryptedFileDownloader.stop(self, err=err)
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager,
|
||||
@@ -173,15 +166,11 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
|
||||
def _close_output(self):
|
||||
self.file_handle, file_handle = None, self.file_handle
|
||||
|
||||
def close_file():
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
return threads.deferToThread(close_file)
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
def _get_write_func(self):
|
||||
def write_func(data):
|
||||
@@ -191,8 +180,8 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
|
||||
|
||||
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
|
||||
super().__init__(peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
|
||||
super().__init__(conf, peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
self.download_directory = hexlify(download_directory.encode())
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
@@ -200,8 +189,9 @@ class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
key = stream_info.raw_info['key']
|
||||
suggested_file_name = stream_info.raw_info['suggested_file_name']
|
||||
return EncryptedFileSaver(
|
||||
stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage, payment_rate_manager,
|
||||
self.wallet, self.download_directory, key=key, stream_name=stream_name, file_name=suggested_file_name
|
||||
self.conf, stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage,
|
||||
payment_rate_manager, self.wallet, self.download_directory, key=key, stream_name=stream_name,
|
||||
file_name=suggested_file_name
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -17,9 +18,8 @@ class EncryptedFileMetadataHandler:
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_initial_blobs(self):
|
||||
blob_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
formatted_infos = self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
defer.returnValue(formatted_infos)
|
||||
blob_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
return self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
|
||||
def final_blob_num(self):
|
||||
return self._final_blob_num
|
||||
|
||||
+455
-600
File diff suppressed because it is too large.
Load diff
@@ -1,109 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import inspect
|
||||
import logging
|
||||
TRACE = 5
|
||||
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
#
|
||||
# _srcfile is used when walking the stack to check when we've got the first
|
||||
# caller stack frame.
|
||||
#
|
||||
if hasattr(sys, 'frozen'): # support for py2exe
|
||||
_srcfile = "logging{}__init__{}".format(os.sep, __file__[-4:])
|
||||
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
|
||||
_srcfile = __file__[:-4] + '.py'
|
||||
else:
|
||||
_srcfile = __file__
|
||||
_srcfile = os.path.normcase(_srcfile)
|
||||
|
||||
|
||||
def findCaller(srcfile=None):
|
||||
"""Returns the filename, line number and function name of the caller"""
|
||||
srcfile = srcfile or _srcfile
|
||||
f = inspect.currentframe()
|
||||
# On some versions of IronPython, currentframe() returns None if
|
||||
# IronPython isn't run with -X:Frames.
|
||||
if f is not None:
|
||||
f = f.f_back
|
||||
rv = "(unknown file)", 0, "(unknown function)"
|
||||
while hasattr(f, "f_code"):
|
||||
co = f.f_code
|
||||
filename = os.path.normcase(co.co_filename)
|
||||
# ignore any function calls that are in this file
|
||||
if filename == srcfile:
|
||||
f = f.f_back
|
||||
continue
|
||||
rv = (filename, f.f_lineno, co.co_name)
|
||||
break
|
||||
return rv
|
||||
|
||||
|
||||
###
|
||||
|
||||
class Logger(logging.Logger):
|
||||
"""A logger that has an extra `fail` method useful for handling twisted failures."""
|
||||
|
||||
def fail(self, callback=None, *args, **kwargs):
|
||||
"""Returns a function to log a failure from an errback.
|
||||
|
||||
The returned function appends the error message and extracts
|
||||
the traceback from `err`.
|
||||
|
||||
Example usage:
|
||||
d.addErrback(log.fail(), 'This is an error message')
|
||||
|
||||
Although odd, making the method call is necessary to extract
|
||||
out useful filename and line number information; otherwise the
|
||||
reported values are from inside twisted's deferred handling
|
||||
code.
|
||||
|
||||
Args:
|
||||
callback: callable to call after making the log. The first argument
|
||||
will be the `err` from the deferred
|
||||
args: extra arguments to pass into `callback`
|
||||
|
||||
Returns: a function that takes the following arguments:
|
||||
err: twisted.python.failure.Failure
|
||||
msg: the message to log, using normal logging string iterpolation.
|
||||
msg_args: the values to substitute into `msg`
|
||||
msg_kwargs: set `level` to change from the default ERROR severity. Other
|
||||
keywoards are treated as normal log kwargs.
|
||||
"""
|
||||
fn, lno, func = findCaller()
|
||||
|
||||
def _fail(err, msg, *msg_args, **msg_kwargs):
|
||||
level = msg_kwargs.pop('level', logging.ERROR)
|
||||
msg += ": %s"
|
||||
msg_args += (err.getErrorMessage(),)
|
||||
exc_info = (err.type, err.value, err.getTracebackObject())
|
||||
record = self.makeRecord(
|
||||
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
|
||||
self.handle(record)
|
||||
if callback:
|
||||
try:
|
||||
return callback(err, *args, **kwargs)
|
||||
except Exception:
|
||||
# log.fail is almost always called within an
|
||||
# errback. If callback fails and we didn't catch
|
||||
# the exception we would need to attach a second
|
||||
# errback to deal with that, which we will almost
|
||||
# never do and then we end up with an unhandled
|
||||
# error that will get swallowed by twisted
|
||||
self.exception('Failed to run callback')
|
||||
|
||||
return _fail
|
||||
|
||||
def trace(self, msg, *args, **kwargs):
|
||||
if self.isEnabledFor(TRACE):
|
||||
self._log(TRACE, msg, args, **kwargs)
|
||||
|
||||
|
||||
def install_logger():
|
||||
current = logging.getLoggerClass()
|
||||
if current is not Logger:
|
||||
logging.setLoggerClass(Logger)
|
||||
logging.addLevelName(TRACE, 'TRACE')
|
||||
+128
-176
@@ -1,132 +1,47 @@
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
import typing
|
||||
|
||||
# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows
|
||||
# This needs to happen before anything else
|
||||
if 'win' in sys.platform:
|
||||
import certifi
|
||||
os.environ['SSL_CERT_FILE'] = certifi.where()
|
||||
|
||||
from twisted.internet import asyncioreactor
|
||||
if 'twisted.internet.reactor' not in sys.modules:
|
||||
asyncioreactor.install()
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
|
||||
# pyinstaller hooks install the default reactor before
|
||||
# any of our code runs, see kivy for similar problem:
|
||||
# https://github.com/kivy/kivy/issues/4182
|
||||
del sys.modules['twisted.internet.reactor']
|
||||
asyncioreactor.install()
|
||||
from twisted.internet import reactor
|
||||
import logging
|
||||
from aiohttp.client_exceptions import ClientConnectorError
|
||||
from requests.exceptions import ConnectionError
|
||||
from docopt import docopt
|
||||
from textwrap import dedent
|
||||
|
||||
from lbrynet import conf, log_support, __name__ as lbrynet_name
|
||||
from lbrynet.utils import check_connection, json_dumps_pretty
|
||||
import aiohttp
|
||||
from lbrynet.extras.compat import force_asyncioreactor_install
|
||||
force_asyncioreactor_install()
|
||||
|
||||
from lbrynet import log_support, __name__ as lbrynet_name, __version__ as lbrynet_version
|
||||
from lbrynet.extras.daemon.loggly_handler import get_loggly_handler
|
||||
from lbrynet.conf import Config, CLIConfig
|
||||
from lbrynet.utils import check_connection
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
from lbrynet.extras.daemon.DaemonConsole import main as daemon_console
|
||||
from lbrynet.extras.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.extras.system_info import get_platform
|
||||
|
||||
log = logging.getLogger(lbrynet_name)
|
||||
|
||||
optional_path_getter_type = typing.Optional[typing.Callable[[], str]]
|
||||
log.addHandler(logging.NullHandler())
|
||||
|
||||
|
||||
def start_daemon(settings: typing.Optional[typing.Dict] = None,
|
||||
console_output: typing.Optional[bool] = True, verbose: typing.Optional[typing.List[str]] = None,
|
||||
data_dir: typing.Optional[str] = None, wallet_dir: typing.Optional[str] = None,
|
||||
download_dir: typing.Optional[str] = None):
|
||||
|
||||
settings = settings or {}
|
||||
conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir)
|
||||
for k, v in settings.items():
|
||||
conf.settings.update({k, v}, data_types=(conf.TYPE_CLI,))
|
||||
|
||||
log_support.configure_logging(conf.settings.get_log_filename(), console_output, verbose)
|
||||
log_support.configure_loggly_handler()
|
||||
log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
if check_connection():
|
||||
daemon = Daemon()
|
||||
daemon.start_listening()
|
||||
reactor.run()
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
def display(data):
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
def start_daemon_with_cli_args(argv=None, data_dir: typing.Optional[str] = None,
|
||||
wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--http-auth", dest="useauth", action="store_true", default=False
|
||||
)
|
||||
parser.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
parser.add_argument(
|
||||
'--version', action="store_true",
|
||||
help='Show daemon version and quit'
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
settings = {}
|
||||
if args.useauth:
|
||||
settings['use_auth_http'] = True
|
||||
|
||||
verbose = None
|
||||
if args.verbose:
|
||||
verbose = args.verbose
|
||||
|
||||
console_output = not args.quiet
|
||||
|
||||
if args.version:
|
||||
print(json_dumps_pretty(get_platform()))
|
||||
return
|
||||
|
||||
return start_daemon(settings, console_output, verbose, data_dir, wallet_dir, download_dir)
|
||||
|
||||
|
||||
async def execute_command(method, params, data_dir: typing.Optional[str] = None,
|
||||
wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None):
|
||||
# this check if the daemon is running or not
|
||||
conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir)
|
||||
api = None
|
||||
try:
|
||||
api = await LBRYAPIClient.get_client()
|
||||
await api.status()
|
||||
except (ClientConnectorError, ConnectionError):
|
||||
if api:
|
||||
await api.session.close()
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
return 1
|
||||
|
||||
# this actually executes the method
|
||||
resp = await api.call(method, params)
|
||||
|
||||
try:
|
||||
await api.session.close()
|
||||
print(json.dumps(resp["result"], indent=2))
|
||||
except KeyError:
|
||||
if resp["error"]["code"] == -32500:
|
||||
print(json.dumps(resp["error"], indent=2))
|
||||
else:
|
||||
print(json.dumps(resp["error"]["message"], indent=2))
|
||||
async def execute_command(conf, method, params):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
message = {'method': method, 'params': params}
|
||||
async with session.get(conf.api_connection_url, json=message) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
display(data['result'])
|
||||
elif 'error' in data:
|
||||
if 'message' in data['error']:
|
||||
display(data['error']['message'])
|
||||
else:
|
||||
display(data['error'])
|
||||
except Exception as e:
|
||||
log.exception('Could not process response from server:', exc_info=e)
|
||||
except aiohttp.ClientConnectionError:
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
|
||||
|
||||
def print_help():
|
||||
@@ -200,82 +115,119 @@ def set_kwargs(parsed_args):
|
||||
return kwargs
|
||||
|
||||
|
||||
def get_argument_parser():
|
||||
main = argparse.ArgumentParser('lbrynet', add_help=False)
|
||||
main.add_argument(
|
||||
'--version', dest='cli_version', action="store_true",
|
||||
help='Show lbrynet CLI version and exit.'
|
||||
)
|
||||
main.add_argument(
|
||||
'-h', '--help', dest='help', action="store_true",
|
||||
help='Show this help message and exit'
|
||||
)
|
||||
CLIConfig.contribute_args(main)
|
||||
sub = main.add_subparsers(dest='command')
|
||||
help = sub.add_parser('help', help='Detailed help for remote commands.')
|
||||
help.add_argument('help_command', nargs='*')
|
||||
start = sub.add_parser('start', help='Start lbrynet server.')
|
||||
start.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
Config.contribute_args(start)
|
||||
api = Daemon.get_api_definitions()
|
||||
for group in sorted(api):
|
||||
group_command = sub.add_parser(group, help=api[group]['doc'])
|
||||
group_command.set_defaults(group_doc=group_command)
|
||||
if group in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
|
||||
continue
|
||||
commands = group_command.add_subparsers(dest='subcommand')
|
||||
for command in api[group]['commands']:
|
||||
commands.add_parser(command['name'], help=command['doc'].strip().splitlines()[0])
|
||||
for deprecated in Daemon.deprecated_methods:
|
||||
group_command = sub.add_parser(deprecated)
|
||||
group_command.add_subparsers(dest='subcommand')
|
||||
return main
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
if not argv:
|
||||
print_help()
|
||||
return 1
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
dir_args = {}
|
||||
if len(argv) > 2:
|
||||
dir_arg_keys = [
|
||||
'data_dir',
|
||||
'wallet_dir',
|
||||
'download_directory'
|
||||
]
|
||||
conf = Config.create_from_arguments(args)
|
||||
|
||||
for arg in argv:
|
||||
found_dir_arg = False
|
||||
for key in dir_arg_keys:
|
||||
if arg.startswith(f'--{key}='):
|
||||
if key in dir_args:
|
||||
print(f"Multiple values provided for '{key}' argument")
|
||||
print_help()
|
||||
return 1
|
||||
dir_args[key] = os.path.expanduser(os.path.expandvars(arg.lstrip(f'--{key}=')))
|
||||
found_dir_arg = True
|
||||
if not found_dir_arg:
|
||||
break
|
||||
argv = argv[len(dir_args):]
|
||||
if args.cli_version:
|
||||
print(f"{lbrynet_name} {lbrynet_version}")
|
||||
return 0
|
||||
|
||||
data_dir = dir_args.get('data_dir')
|
||||
wallet_dir = dir_args.get('wallet_dir')
|
||||
download_dir = dir_args.get('download_directory')
|
||||
elif args.command == 'start':
|
||||
|
||||
for k, v in dir_args.items():
|
||||
if not os.path.isdir(v):
|
||||
print(f"'{data_dir}' is not a directory, cannot use it for {k}")
|
||||
return 1
|
||||
log_support.configure_logging(conf.log_file_path, not args.quiet, args.verbose)
|
||||
|
||||
method, args = argv[0], argv[1:]
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
log.addHandler(loggly_handler)
|
||||
|
||||
if method in ['help', '--help', '-h']:
|
||||
if len(args) == 1:
|
||||
print_help_for_command(args[0])
|
||||
log.debug('Final Settings: %s', conf.settings_dict)
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
daemon = Daemon(conf)
|
||||
|
||||
if check_connection():
|
||||
from twisted.internet import reactor
|
||||
reactor._asyncioEventloop.create_task(daemon.start())
|
||||
reactor.run()
|
||||
else:
|
||||
print_help()
|
||||
return 0
|
||||
log.info("Not connected to internet, unable to start")
|
||||
|
||||
elif method in ['version', '--version', '-v']:
|
||||
print("{lbrynet_name} {lbrynet_version}".format(
|
||||
lbrynet_name=lbrynet_name, **get_platform()
|
||||
))
|
||||
return 0
|
||||
elif args.command == 'help':
|
||||
|
||||
elif method == 'start':
|
||||
sys.exit(start_daemon_with_cli_args(args, data_dir, wallet_dir, download_dir))
|
||||
|
||||
elif method == 'console':
|
||||
sys.exit(daemon_console())
|
||||
|
||||
elif method not in Daemon.callable_methods:
|
||||
if method not in Daemon.deprecated_methods:
|
||||
print(f'{method} is not a valid command.')
|
||||
return 1
|
||||
|
||||
new_method = Daemon.deprecated_methods[method].new_command
|
||||
if new_method is None:
|
||||
print(f"{method} is permanently deprecated and does not have a replacement command.")
|
||||
if args.help_command:
|
||||
method = '_'.join(args.help_command)
|
||||
else:
|
||||
parser.print_help()
|
||||
return 0
|
||||
|
||||
print(f"{method} is deprecated, using {new_method}.")
|
||||
method = new_method
|
||||
if method not in Daemon.callable_methods:
|
||||
print('Invalid command name: {method}')
|
||||
return 1
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
parsed = docopt(fn.__doc__, args)
|
||||
params = set_kwargs(parsed)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(execute_command(method, params, data_dir, wallet_dir, download_dir))
|
||||
fn = Daemon.callable_methods[method]
|
||||
print(fn.__doc__)
|
||||
|
||||
elif args.command is not None:
|
||||
|
||||
if args.command in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
|
||||
method = args.command
|
||||
elif args.subcommand is not None:
|
||||
method = f'{args.command}_{args.subcommand}'
|
||||
else:
|
||||
args.group_doc.print_help()
|
||||
return 0
|
||||
|
||||
if method in Daemon.deprecated_methods:
|
||||
new_method = Daemon.deprecated_methods[method].new_command
|
||||
if new_method is None:
|
||||
print(f"{method} is permanently deprecated and does not have a replacement command.")
|
||||
return 0
|
||||
print(f"{method} is deprecated, using {new_method}.")
|
||||
method = new_method
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
parsed = docopt(fn.__doc__, command_args)
|
||||
params = set_kwargs(parsed)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(execute_command(conf, method, params))
|
||||
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import asyncio
|
||||
|
||||
|
||||
def force_asyncioreactor_install():
|
||||
import sys
|
||||
from twisted.internet import asyncioreactor
|
||||
if 'twisted.internet.reactor' not in sys.modules:
|
||||
asyncioreactor.install()
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
|
||||
# pyinstaller hooks install the default reactor before
|
||||
# any of our code runs, see kivy for similar problem:
|
||||
# https://github.com/kivy/kivy/issues/4182
|
||||
del sys.modules['twisted.internet.reactor']
|
||||
asyncioreactor.install()
|
||||
|
||||
|
||||
def d2f(deferred):
|
||||
return deferred.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def f2d(future):
|
||||
from twisted.internet import defer
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
@@ -1,6 +1,8 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted._threads import AlreadyQuit
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -26,6 +28,7 @@ class Component(metaclass=ComponentType):
|
||||
component_name = None
|
||||
|
||||
def __init__(self, component_manager):
|
||||
self.conf: Config = component_manager.conf
|
||||
self.component_manager = component_manager
|
||||
self._running = False
|
||||
|
||||
@@ -36,37 +39,35 @@ class Component(metaclass=ComponentType):
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup(self):
|
||||
async def _setup(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.start)
|
||||
result = await self.start()
|
||||
self._running = True
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||
log.exception("Error setting up %s", self.component_name or self.__class__.__name__)
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop(self):
|
||||
async def _stop(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.stop)
|
||||
result = await self.stop()
|
||||
self._running = False
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from lbrynet.p2p.Error import ComponentStartConditionNotMet
|
||||
from lbrynet.extras.daemon.PeerManager import PeerManager
|
||||
from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
|
||||
from lbrynet.conf import Config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -34,8 +35,9 @@ class RequiredCondition(metaclass=RequiredConditionType):
|
||||
class ComponentManager:
|
||||
default_component_classes = {}
|
||||
|
||||
def __init__(self, reactor=None, analytics_manager=None, skip_components=None,
|
||||
def __init__(self, conf: Config, reactor=None, analytics_manager=None, skip_components=None,
|
||||
peer_manager=None, peer_finder=None, **override_components):
|
||||
self.conf = conf
|
||||
self.skip_components = skip_components or []
|
||||
self.reactor = reactor
|
||||
self.component_classes = {}
|
||||
@@ -55,6 +57,7 @@ class ComponentManager:
|
||||
|
||||
for component_class in self.component_classes.values():
|
||||
self.components.add(component_class(self))
|
||||
self.daemon = None
|
||||
|
||||
def evaluate_condition(self, condition_name):
|
||||
if condition_name not in RegisteredConditions.conditions:
|
||||
@@ -110,13 +113,8 @@ class ComponentManager:
|
||||
steps.reverse()
|
||||
return steps
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self, **callbacks):
|
||||
"""
|
||||
Start Components in sequence sorted by requirements
|
||||
|
||||
:return: (defer.Deferred)
|
||||
"""
|
||||
async def setup(self, **callbacks):
|
||||
""" Start Components in sequence sorted by requirements """
|
||||
for component_name, cb in callbacks.items():
|
||||
if component_name not in self.component_classes:
|
||||
if component_name not in self.skip_components:
|
||||
@@ -124,19 +122,22 @@ class ComponentManager:
|
||||
if not callable(cb):
|
||||
raise ValueError("%s is not callable" % cb)
|
||||
|
||||
def _setup(component):
|
||||
async def _setup(component):
|
||||
await component._setup()
|
||||
if component.component_name in callbacks:
|
||||
d = component._setup()
|
||||
d.addCallback(callbacks[component.component_name], component)
|
||||
return d
|
||||
return component._setup()
|
||||
maybe_coro = callbacks[component.component_name](component)
|
||||
if asyncio.iscoroutine(maybe_coro):
|
||||
asyncio.create_task(maybe_coro)
|
||||
|
||||
stages = self.sort_components()
|
||||
for stage in stages:
|
||||
yield defer.DeferredList([_setup(component) for component in stage if not component.running])
|
||||
needing_start = [
|
||||
_setup(component) for component in stage if not component.running
|
||||
]
|
||||
if needing_start:
|
||||
await asyncio.wait(needing_start)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
"""
|
||||
Stop Components in reversed startup order
|
||||
|
||||
@@ -144,7 +145,11 @@ class ComponentManager:
|
||||
"""
|
||||
stages = self.sort_components(reverse=True)
|
||||
for stage in stages:
|
||||
yield defer.DeferredList([component._stop() for component in stage if component.running])
|
||||
needing_stop = [
|
||||
component._stop() for component in stage if component.running
|
||||
]
|
||||
if needing_stop:
|
||||
await asyncio.wait(needing_stop)
|
||||
|
||||
def all_components_running(self, *component_names):
|
||||
"""
|
||||
|
||||
+138
-177
@@ -1,33 +1,34 @@
|
||||
import os
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import logging
|
||||
import treq
|
||||
import json
|
||||
import math
|
||||
import binascii
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
from twisted.internet import defer, threads, reactor, error, task
|
||||
from twisted.internet import defer, reactor, error
|
||||
|
||||
from aioupnp import __version__ as aioupnp_version
|
||||
from aioupnp.upnp import UPnP
|
||||
from aioupnp.fault import UPnPError
|
||||
|
||||
import lbrynet.schema
|
||||
from lbrynet import conf
|
||||
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.extras.compat import d2f
|
||||
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
|
||||
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.Component import Component
|
||||
from lbrynet.extras.daemon.ExchangeRateManager import ExchangeRateManager
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
|
||||
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.wallet import Network
|
||||
from lbrynet.utils import DeferredDict, generate_id
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
|
||||
from lbrynet.p2p.RateLimiter import RateLimiter
|
||||
from lbrynet.p2p.BlobManager import DiskBlobManager
|
||||
@@ -55,22 +56,23 @@ RATE_LIMITER_COMPONENT = "rate_limiter"
|
||||
PAYMENT_RATE_COMPONENT = "payment_rate_manager"
|
||||
|
||||
|
||||
def from_future(coroutine: asyncio.coroutine) -> defer.Deferred:
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(coroutine))
|
||||
async def gather_dict(tasks: dict):
|
||||
async def wait_value(key, value):
|
||||
return key, await value
|
||||
return dict(await asyncio.gather(*(
|
||||
wait_value(*kv) for kv in tasks.items()
|
||||
)))
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
async def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
buf = []
|
||||
response = yield treq.get("https://api.lbry.io/ip")
|
||||
yield treq.collect(response, buf.append)
|
||||
parsed = json.loads(b"".join(buf).decode())
|
||||
if parsed['success']:
|
||||
return parsed['data']['ip']
|
||||
return
|
||||
except Exception as err:
|
||||
return
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get("https://api.lbry.io/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
@@ -88,26 +90,24 @@ class DatabaseComponent(Component):
|
||||
def get_current_db_revision():
|
||||
return 9
|
||||
|
||||
@staticmethod
|
||||
def get_revision_filename():
|
||||
return conf.settings.get_db_revision_filename()
|
||||
@property
|
||||
def revision_filename(self):
|
||||
return self.component_manager.daemon.db_revision_file_path
|
||||
|
||||
@staticmethod
|
||||
def _write_db_revision_file(version_num):
|
||||
with open(conf.settings.get_db_revision_filename(), mode='w') as db_revision:
|
||||
def _write_db_revision_file(self, version_num):
|
||||
with open(self.revision_filename, mode='w') as db_revision:
|
||||
db_revision.write(str(version_num))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
# check directories exist, create them if they don't
|
||||
log.info("Loading databases")
|
||||
|
||||
if not os.path.exists(self.get_revision_filename()):
|
||||
if not os.path.exists(self.revision_filename):
|
||||
log.warning("db_revision file not found. Creating it")
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
|
||||
# check the db migration and run any needed migrations
|
||||
with open(self.get_revision_filename(), "r") as revision_read_handle:
|
||||
with open(self.revision_filename, "r") as revision_read_handle:
|
||||
old_revision = int(revision_read_handle.read().strip())
|
||||
|
||||
if old_revision > self.get_current_db_revision():
|
||||
@@ -117,19 +117,19 @@ class DatabaseComponent(Component):
|
||||
if old_revision < self.get_current_db_revision():
|
||||
from lbrynet.extras.daemon.migrator import dbmigrator
|
||||
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
|
||||
yield threads.deferToThread(
|
||||
dbmigrator.migrate_db, conf.settings.data_dir, old_revision, self.get_current_db_revision()
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
|
||||
)
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.info("Finished upgrading the databases.")
|
||||
|
||||
# start SQLiteStorage
|
||||
self.storage = SQLiteStorage(conf.settings.data_dir)
|
||||
yield self.storage.setup()
|
||||
self.storage = SQLiteStorage(
|
||||
self.conf, os.path.join(self.conf.data_dir, "lbrynet.sqlite")
|
||||
)
|
||||
await self.storage.open()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.storage.stop()
|
||||
async def stop(self):
|
||||
await self.storage.close()
|
||||
self.storage = None
|
||||
|
||||
|
||||
@@ -142,9 +142,9 @@ class HeadersComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.headers_dir = os.path.join(conf.settings.wallet_dir, 'lbc_mainnet')
|
||||
self.headers_dir = os.path.join(self.conf.wallet_dir, 'lbc_mainnet')
|
||||
self.headers_file = os.path.join(self.headers_dir, 'headers')
|
||||
self.old_file = os.path.join(conf.settings.wallet_dir, 'blockchain_headers')
|
||||
self.old_file = os.path.join(self.conf.wallet_dir, 'blockchain_headers')
|
||||
self._downloading_headers = None
|
||||
self._headers_progress_percent = 0
|
||||
|
||||
@@ -152,7 +152,7 @@ class HeadersComponent(Component):
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {} if not self._downloading_headers else {
|
||||
'downloading_headers': self._downloading_headers,
|
||||
'download_progress': self._headers_progress_percent
|
||||
@@ -203,8 +203,8 @@ class HeadersComponent(Component):
|
||||
async def get_remote_height(self):
|
||||
ledger = SimpleNamespace()
|
||||
ledger.config = {
|
||||
'default_servers': conf.settings['lbryum_servers'],
|
||||
'data_path': conf.settings.wallet_dir
|
||||
'default_servers': self.conf.lbryum_servers,
|
||||
'data_path': self.conf.wallet_dir
|
||||
}
|
||||
net = Network(ledger)
|
||||
first_connection = net.on_connected.first
|
||||
@@ -215,10 +215,10 @@ class HeadersComponent(Component):
|
||||
return remote_height
|
||||
|
||||
async def should_download_headers_from_s3(self):
|
||||
if conf.settings['blockchain_name'] != "lbrycrd_main":
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
return False
|
||||
self._check_header_file_integrity()
|
||||
s3_headers_depth = conf.settings['s3_headers_depth']
|
||||
s3_headers_depth = self.conf.s3_headers_depth
|
||||
if not s3_headers_depth:
|
||||
return False
|
||||
local_height = self.local_header_file_height()
|
||||
@@ -230,10 +230,10 @@ class HeadersComponent(Component):
|
||||
|
||||
def _check_header_file_integrity(self):
|
||||
# TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
|
||||
if conf.settings['blockchain_name'] != "lbrycrd_main":
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
return
|
||||
hashsum = sha256()
|
||||
checksum_height, checksum = conf.settings['HEADERS_FILE_SHA256_CHECKSUM']
|
||||
checksum_height, checksum = HEADERS_FILE_SHA256_CHECKSUM
|
||||
checksum_length_in_bytes = checksum_height * HEADER_SIZE
|
||||
if self.local_header_file_size() < checksum_length_in_bytes:
|
||||
return
|
||||
@@ -250,33 +250,23 @@ class HeadersComponent(Component):
|
||||
with open(self.headers_file, "rb+") as headers_file:
|
||||
headers_file.truncate(checksum_length_in_bytes)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
conf.settings.ensure_wallet_dir()
|
||||
async def start(self):
|
||||
if not os.path.exists(self.headers_dir):
|
||||
os.mkdir(self.headers_dir)
|
||||
if os.path.exists(self.old_file):
|
||||
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
|
||||
os.rename(self.old_file, self.headers_file)
|
||||
self._downloading_headers = yield f2d(self.should_download_headers_from_s3())
|
||||
self._downloading_headers = await self.should_download_headers_from_s3()
|
||||
if self._downloading_headers:
|
||||
try:
|
||||
yield self.fetch_headers_from_s3()
|
||||
await d2f(self.fetch_headers_from_s3())
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self._downloading_headers = False
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
def d2f(deferred):
|
||||
return deferred.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def f2d(future):
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class WalletComponent(Component):
|
||||
@@ -291,7 +281,7 @@ class WalletComponent(Component):
|
||||
def component(self):
|
||||
return self.wallet_manager
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
if self.wallet_manager and self.running:
|
||||
local_height = self.wallet_manager.network.get_local_height()
|
||||
remote_height = self.wallet_manager.network.get_server_height()
|
||||
@@ -304,19 +294,16 @@ class WalletComponent(Component):
|
||||
'is_locked': not self.wallet_manager.is_wallet_unlocked,
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
conf.settings.ensure_wallet_dir()
|
||||
async def start(self):
|
||||
log.info("Starting torba wallet")
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = conf.settings['blockchain_name']
|
||||
self.wallet_manager = yield f2d(LbryWalletManager.from_lbrynet_config(conf.settings, storage))
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = self.conf.blockchain_name
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf, storage)
|
||||
self.wallet_manager.old_db = storage
|
||||
yield f2d(self.wallet_manager.start())
|
||||
await self.wallet_manager.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield f2d(self.wallet_manager.stop())
|
||||
async def stop(self):
|
||||
await self.wallet_manager.stop()
|
||||
self.wallet_manager = None
|
||||
|
||||
|
||||
@@ -339,20 +326,17 @@ class BlobComponent(Component):
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_node:
|
||||
datastore = dht_node._dataStore
|
||||
self.blob_manager = DiskBlobManager(os.path.join(conf.settings.data_dir, "blobfiles"), storage, datastore)
|
||||
self.blob_manager = DiskBlobManager(os.path.join(self.conf.data_dir, "blobfiles"), storage, datastore)
|
||||
return self.blob_manager.setup()
|
||||
|
||||
def stop(self):
|
||||
return self.blob_manager.stop()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
count = 0
|
||||
if self.blob_manager:
|
||||
count = yield self.blob_manager.storage.count_finished_blobs()
|
||||
defer.returnValue({
|
||||
'finished_blobs': count
|
||||
})
|
||||
count = await self.blob_manager.storage.count_finished_blobs()
|
||||
return {'finished_blobs': count}
|
||||
|
||||
|
||||
class DHTComponent(Component):
|
||||
@@ -370,41 +354,39 @@ class DHTComponent(Component):
|
||||
def component(self):
|
||||
return self.dht_node
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {
|
||||
'node_id': binascii.hexlify(conf.settings.get_node_id()),
|
||||
'node_id': binascii.hexlify(self.component_manager.daemon.node_id),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", conf.settings["peer_port"])
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", conf.settings["dht_node_port"])
|
||||
node_id = conf.settings.get_node_id()
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
|
||||
node_id = self.component_manager.daemon.node_id
|
||||
if node_id is None:
|
||||
node_id = generate_id()
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
external_ip = yield get_external_ip()
|
||||
external_ip = await get_external_ip()
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
self.dht_node = Node(
|
||||
node_id=node_id,
|
||||
udpPort=conf.settings['dht_node_port'],
|
||||
udpPort=self.conf.dht_node_port,
|
||||
externalUDPPort=self.external_udp_port,
|
||||
externalIP=external_ip,
|
||||
peerPort=self.external_peer_port
|
||||
)
|
||||
|
||||
yield self.dht_node.start(conf.settings['known_dht_nodes'], block_on_join=False)
|
||||
await d2f(self.dht_node.start(self.conf.known_dht_nodes, block_on_join=False))
|
||||
log.info("Started the dht")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.dht_node.stop()
|
||||
return d2f(self.dht_node.stop())
|
||||
|
||||
|
||||
class HashAnnouncerComponent(Component):
|
||||
@@ -419,18 +401,16 @@ class HashAnnouncerComponent(Component):
|
||||
def component(self):
|
||||
return self.hash_announcer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = DHTHashAnnouncer(dht_node, storage)
|
||||
yield self.hash_announcer.start()
|
||||
self.hash_announcer = DHTHashAnnouncer(self.conf, dht_node, storage)
|
||||
self.hash_announcer.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.hash_announcer.stop()
|
||||
self.hash_announcer.stop()
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {
|
||||
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.hash_queue)
|
||||
}
|
||||
@@ -447,13 +427,11 @@ class RateLimiterComponent(Component):
|
||||
def component(self):
|
||||
return self.rate_limiter
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
self.rate_limiter.start()
|
||||
return defer.succeed(None)
|
||||
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
self.rate_limiter.stop()
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class PaymentRateComponent(Component):
|
||||
@@ -467,11 +445,11 @@ class PaymentRateComponent(Component):
|
||||
def component(self):
|
||||
return self.payment_rate_manager
|
||||
|
||||
def start(self):
|
||||
return defer.succeed(None)
|
||||
async def start(self):
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class FileManagerComponent(Component):
|
||||
@@ -487,14 +465,13 @@ class FileManagerComponent(Component):
|
||||
def component(self):
|
||||
return self.file_manager
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
if not self.file_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.file_manager.lbry_files)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
@@ -504,25 +481,26 @@ class FileManagerComponent(Component):
|
||||
sd_identifier = StreamDescriptorIdentifier()
|
||||
add_lbry_file_to_sd_identifier(sd_identifier)
|
||||
file_saver_factory = EncryptedFileSaverFactory(
|
||||
self.conf,
|
||||
self.component_manager.peer_finder,
|
||||
rate_limiter,
|
||||
blob_manager,
|
||||
storage,
|
||||
wallet,
|
||||
conf.settings.download_dir
|
||||
self.conf.download_dir
|
||||
)
|
||||
yield sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
|
||||
sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
|
||||
|
||||
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
|
||||
log.info('Starting the file manager')
|
||||
self.file_manager = EncryptedFileManager(self.component_manager.peer_finder, rate_limiter, blob_manager, wallet,
|
||||
payment_rate_manager, storage, sd_identifier)
|
||||
yield self.file_manager.setup()
|
||||
log.info('Done setting up file manager')
|
||||
self.file_manager = EncryptedFileManager(
|
||||
self.conf, self.component_manager.peer_finder, rate_limiter,
|
||||
blob_manager, wallet, payment_rate_manager, storage, sd_identifier
|
||||
)
|
||||
return self.file_manager.setup()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.file_manager.stop()
|
||||
return d2f(self.file_manager.stop())
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -538,11 +516,10 @@ class PeerProtocolServerComponent(Component):
|
||||
def component(self):
|
||||
return self.lbry_server_port
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
peer_port = conf.settings['peer_port']
|
||||
peer_port = self.conf.peer_port
|
||||
query_handlers = {
|
||||
handler.get_primary_query_identifier(): handler for handler in [
|
||||
BlobRequestHandlerFactory(
|
||||
@@ -562,7 +539,7 @@ class PeerProtocolServerComponent(Component):
|
||||
try:
|
||||
log.info("Peer protocol listening on TCP %i (ext port %i)", peer_port,
|
||||
upnp.upnp_redirects.get("TCP", peer_port))
|
||||
self.lbry_server_port = yield reactor.listenTCP(peer_port, server_factory)
|
||||
self.lbry_server_port = reactor.listenTCP(peer_port, server_factory)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. Visit lbry.io/faq/how-to-change-port for"
|
||||
@@ -570,12 +547,11 @@ class PeerProtocolServerComponent(Component):
|
||||
log.error("%s", traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running on your computer." % str(e))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
if self.lbry_server_port is not None:
|
||||
self.lbry_server_port, old_port = None, self.lbry_server_port
|
||||
log.info('Stop listening on port %s', old_port.port)
|
||||
yield old_port.stopListening()
|
||||
await d2f(old_port.stopListening())
|
||||
|
||||
|
||||
class ReflectorComponent(Component):
|
||||
@@ -584,32 +560,30 @@ class ReflectorComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.reflector_server_port = conf.settings['reflector_port']
|
||||
self.reflector_server_port = self.conf.reflector_port
|
||||
self.reflector_server = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.reflector_server
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
log.info("Starting reflector server")
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
|
||||
reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager)
|
||||
try:
|
||||
self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory)
|
||||
self.reflector_server = await d2f(reactor.listenTCP(self.reflector_server_port, reflector_factory))
|
||||
log.info('Started reflector on port %s', self.reflector_server_port)
|
||||
except error.CannotListenError as e:
|
||||
log.exception("Couldn't bind reflector to port %d", self.reflector_server_port)
|
||||
raise ValueError(f"{e} lbrynet may already be running on your computer.")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
if self.reflector_server is not None:
|
||||
log.info("Stopping reflector server")
|
||||
self.reflector_server, p = None, self.reflector_server
|
||||
yield p.stopListening
|
||||
await d2f(p.stopListening())
|
||||
|
||||
|
||||
class UPnPComponent(Component):
|
||||
@@ -617,35 +591,29 @@ class UPnPComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self._int_peer_port = conf.settings['peer_port']
|
||||
self._int_dht_node_port = conf.settings['dht_node_port']
|
||||
self.use_upnp = conf.settings['use_upnp']
|
||||
self._int_peer_port = self.conf.peer_port
|
||||
self._int_dht_node_port = self.conf.dht_node_port
|
||||
self.use_upnp = self.conf.use_upnp
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
self.external_ip = None
|
||||
self._maintain_redirects_lc = task.LoopingCall(self._maintain_redirects)
|
||||
self._maintain_redirects_lc.clock = self.component_manager.reactor
|
||||
self._maintain_redirects_task = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup_redirects(self):
|
||||
d = {}
|
||||
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
|
||||
upnp_redirects = yield DeferredDict(d)
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
async def _repeatedly_maintain_redirects(self, now=True):
|
||||
while True:
|
||||
if now:
|
||||
await self._maintain_redirects()
|
||||
await asyncio.sleep(360)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _maintain_redirects(self):
|
||||
async def _maintain_redirects(self):
|
||||
# setup the gateway if necessary
|
||||
if not self.upnp:
|
||||
try:
|
||||
self.upnp = yield from_future(UPnP.discover())
|
||||
self.upnp = await UPnP.discover()
|
||||
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
except Exception as err:
|
||||
log.warning("upnp discovery failed: %s", err)
|
||||
@@ -655,7 +623,7 @@ class UPnPComponent(Component):
|
||||
external_ip = None
|
||||
if self.upnp:
|
||||
try:
|
||||
external_ip = yield from_future(self.upnp.get_external_ip())
|
||||
external_ip = await self.upnp.get_external_ip()
|
||||
if external_ip != "0.0.0.0" and not self.external_ip:
|
||||
log.info("got external ip from UPnP: %s", external_ip)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
@@ -663,7 +631,7 @@ class UPnPComponent(Component):
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
external_ip = yield get_external_ip()
|
||||
external_ip = await get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
self.external_ip = external_ip
|
||||
@@ -674,10 +642,10 @@ class UPnPComponent(Component):
|
||||
log.info("add UPnP port mappings")
|
||||
d = {}
|
||||
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
|
||||
d["TCP"] = self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
|
||||
upnp_redirects = yield DeferredDict(d)
|
||||
d["UDP"] = self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
upnp_redirects = await gather_dict(d)
|
||||
log.info("set up redirects: %s", upnp_redirects)
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
@@ -685,7 +653,7 @@ class UPnPComponent(Component):
|
||||
return self._maintain_redirects()
|
||||
elif self.upnp: # check existing redirects are still active
|
||||
found = set()
|
||||
mappings = yield from_future(self.upnp.get_redirects())
|
||||
mappings = await self.upnp.get_redirects()
|
||||
for mapping in mappings:
|
||||
proto = mapping['NewProtocol']
|
||||
if proto in self.upnp_redirects and mapping['NewExternalPort'] == self.upnp_redirects[proto]:
|
||||
@@ -693,18 +661,14 @@ class UPnPComponent(Component):
|
||||
found.add(proto)
|
||||
if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
udp_port = yield from_future(
|
||||
self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
)
|
||||
udp_port = await self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
self.upnp_redirects['UDP'] = udp_port
|
||||
log.info("refreshed upnp redirect for dht port: %i", udp_port)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
del self.upnp_redirects['UDP']
|
||||
if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
tcp_port = yield from_future(
|
||||
self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
)
|
||||
tcp_port = await self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
self.upnp_redirects['TCP'] = tcp_port
|
||||
log.info("refreshed upnp redirect for peer port: %i", tcp_port)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
@@ -715,14 +679,13 @@ class UPnPComponent(Component):
|
||||
if self.upnp_redirects:
|
||||
log.debug("upnp redirects are still active")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = yield get_external_ip()
|
||||
self.external_ip = await get_external_ip()
|
||||
return
|
||||
success = False
|
||||
yield self._maintain_redirects()
|
||||
await self._maintain_redirects()
|
||||
if self.upnp:
|
||||
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
|
||||
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
|
||||
@@ -733,18 +696,18 @@ class UPnPComponent(Component):
|
||||
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, self.get_status())
|
||||
self._maintain_redirects_lc.start(360, now=False)
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
|
||||
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
|
||||
|
||||
def stop(self):
|
||||
if self._maintain_redirects_lc.running:
|
||||
self._maintain_redirects_lc.stop()
|
||||
return defer.DeferredList(
|
||||
[from_future(self.upnp.delete_port_mapping(port, protocol))
|
||||
for protocol, port in self.upnp_redirects.items()]
|
||||
)
|
||||
async def stop(self):
|
||||
if self.upnp_redirects:
|
||||
await asyncio.wait([
|
||||
self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
|
||||
])
|
||||
if self._maintain_redirects_task is not None and not self._maintain_redirects_task.done():
|
||||
self._maintain_redirects_task.cancel()
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {
|
||||
'aioupnp_version': aioupnp_version,
|
||||
'redirects': self.upnp_redirects,
|
||||
@@ -766,10 +729,8 @@ class ExchangeRateManagerComponent(Component):
|
||||
def component(self):
|
||||
return self.exchange_rate_manager
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
yield self.exchange_rate_manager.start()
|
||||
async def start(self):
|
||||
self.exchange_rate_manager.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.exchange_rate_manager.stop()
|
||||
async def stop(self):
|
||||
self.exchange_rate_manager.stop()
|
||||
+687
-425
File diff suppressed because it is too large.
Load diff
@@ -1,230 +0,0 @@
|
||||
import sys
|
||||
import code
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging.handlers
|
||||
from twisted.internet import defer, reactor, threads
|
||||
from aiohttp import client_exceptions
|
||||
|
||||
from lbrynet import utils, conf, log_support
|
||||
from lbrynet.extras.daemon import analytics
|
||||
from lbrynet.extras.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
if sys.platform.startswith('darwin') or sys.platform.startswith('linux'):
|
||||
def color(msg, c="white"):
|
||||
_colors = {
|
||||
"normal": (0, 37),
|
||||
"underlined": (2, 37),
|
||||
"red": (1, 31),
|
||||
"green": (1, 32),
|
||||
"yellow": (1, 33),
|
||||
"blue": (1, 33),
|
||||
"magenta": (1, 34),
|
||||
"cyan": (1, 35),
|
||||
"white": (1, 36),
|
||||
"grey": (1, 37)
|
||||
}
|
||||
i, j = _colors[c]
|
||||
return "\033[%i;%i;40m%s\033[0m" % (i, j, msg)
|
||||
|
||||
|
||||
logo = """\
|
||||
╓▄█▄ç
|
||||
,▄█▓▓▀▀▀▓▓▓▌▄,
|
||||
▄▄▓▓▓▀¬ ╙▀█▓▓▓▄▄
|
||||
,▄█▓▓▀▀ ^▀▀▓▓▓▌▄,
|
||||
▄█▓▓█▀` ╙▀█▓▓▓▄▄
|
||||
╓▄▓▓▓▀╙ ▀▀▓▓▓▌▄,
|
||||
▄█▓▓█▀ ╙▀▓▓
|
||||
╓▄▓▓▓▀` ▄█▓▓▓▀
|
||||
▓▓█▀ ,▄▓▓▓▀╙
|
||||
▓▓m ╟▌▄, ▄█▓▓█▀ ,,╓µ
|
||||
▓▓m ^▀█▓▓▓▄▄ ╓▄▓▓▓▀╙ █▓▓▓▓▓▀
|
||||
▓▓Q '▀▀▓▓▓▌▄, ,▄█▓▓█▀ ▄█▓▓▓▓▓▀
|
||||
▀▓▓▓▌▄, ╙▀█▓▓█▄╗ ╓▄▓▓▓▀ ╓▄▓▓▓▀▀ ▀▀
|
||||
╙▀█▓▓█▄╗ ^▀▀▓▓▓▌▄▄█▓▓▀▀ ▄█▓▓█▀`
|
||||
'▀▀▓▓▓▌▄, ╙▀██▀` ╓▄▓▓▓▀╙
|
||||
╙▀█▓▓█▄╥ ,▄█▓▓▀▀
|
||||
└▀▀▓▓▓▌▄ ▄▒▓▓▓▀╙
|
||||
╙▀█▓▓█▓▓▓▀▀
|
||||
╙▀▀`
|
||||
|
||||
"""
|
||||
else:
|
||||
def color(msg, c=None):
|
||||
return msg
|
||||
|
||||
logo = """\
|
||||
'.
|
||||
++++.
|
||||
+++,;+++,
|
||||
:+++ :+++:
|
||||
+++ ,+++;
|
||||
'++; .+++'
|
||||
`+++ `++++
|
||||
+++. `++++
|
||||
;+++ ++++
|
||||
+++ +++
|
||||
+++: '+
|
||||
,+++ +++
|
||||
+++` +++:
|
||||
`+' ,+++
|
||||
`+ + +++
|
||||
`+ +++ '++' :'+++:
|
||||
`+ ++++ `+++ ++++
|
||||
`+ ++++ +++. :+++'
|
||||
`+, ++++ ;+++ +++++
|
||||
`+++, ++++ +++ +++; +
|
||||
,+++, ++++ +++: .+++
|
||||
,+++: '++++++ +++`
|
||||
,+++: '++ '++'
|
||||
,+++: `+++
|
||||
.+++; +++,
|
||||
.+++; ;+++
|
||||
.+++; +++
|
||||
`+++++:
|
||||
`++
|
||||
|
||||
"""
|
||||
|
||||
welcometext = """\
|
||||
For a list of available commands:
|
||||
>>>help()
|
||||
|
||||
To see the documentation for a given command:
|
||||
>>>help("resolve")
|
||||
|
||||
To exit:
|
||||
>>>exit()
|
||||
"""
|
||||
|
||||
welcome = "{:*^60}\n".format(" Welcome to the lbrynet interactive console! ")
|
||||
welcome += "\n".join([f"{w:<60}" for w in welcometext.splitlines()])
|
||||
welcome += "\n%s" % ("*" * 60)
|
||||
welcome = color(welcome, "grey")
|
||||
banner = color(logo, "green") + color(welcome, "grey")
|
||||
|
||||
|
||||
def get_methods(daemon):
|
||||
locs = {}
|
||||
|
||||
def wrapped(name, fn):
|
||||
client = LBRYAPIClient.get_client()
|
||||
_fn = getattr(client, name)
|
||||
_fn.__doc__ = fn.__doc__
|
||||
return {name: _fn}
|
||||
|
||||
for method_name, method in daemon.callable_methods.items():
|
||||
locs.update(wrapped(method_name, method))
|
||||
return locs
|
||||
|
||||
|
||||
def run_terminal(callable_methods, started_daemon, quiet=False):
|
||||
locs = {}
|
||||
locs.update(callable_methods)
|
||||
|
||||
def help(method_name=None):
|
||||
if not method_name:
|
||||
print("Available api functions: ")
|
||||
for name in callable_methods:
|
||||
print("\t%s" % name)
|
||||
return
|
||||
if method_name not in callable_methods:
|
||||
print("\"%s\" is not a recognized api function")
|
||||
return
|
||||
print(callable_methods[method_name].__doc__)
|
||||
return
|
||||
|
||||
locs.update({'help': help})
|
||||
|
||||
if started_daemon:
|
||||
def exit(status=None):
|
||||
if not quiet:
|
||||
print("Stopping lbrynet-daemon...")
|
||||
callable_methods['daemon_stop']()
|
||||
return sys.exit(status)
|
||||
|
||||
locs.update({'exit': exit})
|
||||
else:
|
||||
def exit(status=None):
|
||||
try:
|
||||
reactor.callLater(0, reactor.stop)
|
||||
except Exception as err:
|
||||
print(f"error stopping reactor: {err}")
|
||||
return sys.exit(status)
|
||||
|
||||
locs.update({'exit': exit})
|
||||
|
||||
code.interact(banner if not quiet else "", local=locs)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_server_and_listen(use_auth, analytics_manager, quiet):
|
||||
log_support.configure_console()
|
||||
logging.getLogger("lbrynet").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("lbryum").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("requests").setLevel(logging.CRITICAL)
|
||||
|
||||
analytics_manager.send_server_startup()
|
||||
yield Daemon().start_listening()
|
||||
|
||||
|
||||
def threaded_terminal(started_daemon, quiet):
|
||||
callable_methods = get_methods(Daemon)
|
||||
d = threads.deferToThread(run_terminal, callable_methods, started_daemon, quiet)
|
||||
d.addErrback(lambda err: err.trap(SystemExit))
|
||||
d.addErrback(log.exception)
|
||||
|
||||
|
||||
async def start_lbrynet_console(quiet, use_existing_daemon, useauth):
|
||||
if not utils.check_connection():
|
||||
print("Not connected to internet, unable to start")
|
||||
raise Exception("Not connected to internet, unable to start")
|
||||
if not quiet:
|
||||
print("Starting lbrynet-console...")
|
||||
try:
|
||||
await LBRYAPIClient.get_client().status()
|
||||
d = defer.succeed(False)
|
||||
if not quiet:
|
||||
print("lbrynet-daemon is already running, connecting to it...")
|
||||
except client_exceptions.ClientConnectorError:
|
||||
if not use_existing_daemon:
|
||||
if not quiet:
|
||||
print("Starting lbrynet-daemon...")
|
||||
analytics_manager = analytics.Manager.new_instance()
|
||||
d = start_server_and_listen(useauth, analytics_manager, quiet)
|
||||
else:
|
||||
raise Exception("cannot connect to an existing daemon instance, "
|
||||
"and set to not start a new one")
|
||||
d.addCallback(threaded_terminal, quiet)
|
||||
d.addErrback(log.exception)
|
||||
|
||||
|
||||
def main():
|
||||
conf.initialize_settings()
|
||||
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
|
||||
parser.add_argument(
|
||||
"--use_existing_daemon",
|
||||
help="Start lbrynet-daemon if it isn't already running",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="use_existing_daemon"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--quiet", dest="quiet", action="store_true", default=False
|
||||
)
|
||||
parser.add_argument(
|
||||
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
|
||||
)
|
||||
args = parser.parse_args()
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(start_lbrynet_console(args.quiet, args.use_existing_daemon, args.useauth))
|
||||
reactor.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -2,7 +2,7 @@ import logging
|
||||
import os
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.schema.fee import Fee
|
||||
|
||||
from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
|
||||
@@ -11,7 +11,7 @@ from lbrynet.p2p.StreamDescriptor import download_sd_blob
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from torba.client.constants import COIN
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.extras.daemon.Components import f2d
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
INITIALIZING_CODE = 'initializing'
|
||||
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
|
||||
@@ -31,17 +31,18 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GetStream:
|
||||
def __init__(self, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder, rate_limiter,
|
||||
payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None, timeout=None,
|
||||
reactor=None):
|
||||
def __init__(self, conf: Config, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder,
|
||||
rate_limiter, payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None,
|
||||
timeout=None, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.conf = conf
|
||||
self.reactor = reactor
|
||||
self.timeout = timeout or conf.settings['download_timeout']
|
||||
self.data_rate = data_rate or conf.settings['data_rate']
|
||||
self.max_key_fee = max_key_fee or conf.settings['max_key_fee'][1]
|
||||
self.disable_max_key_fee = disable_max_key_fee or conf.settings['disable_max_key_fee']
|
||||
self.download_directory = conf.settings.download_dir
|
||||
self.timeout = timeout or conf.download_timeout
|
||||
self.data_rate = data_rate or conf.data_rate
|
||||
self.max_key_fee = max_key_fee or conf.max_key_fee
|
||||
self.disable_max_key_fee = disable_max_key_fee or conf.disable_max_key_fee
|
||||
self.download_directory = conf.download_dir
|
||||
self.timeout_counter = 0
|
||||
self.code = None
|
||||
self.sd_hash = None
|
||||
@@ -153,8 +154,8 @@ class GetStream:
|
||||
@defer.inlineCallbacks
|
||||
def _download_sd_blob(self):
|
||||
sd_blob = yield download_sd_blob(
|
||||
self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
|
||||
self.wallet, self.timeout, conf.settings['download_mirrors']
|
||||
self.conf, self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
|
||||
self.wallet, self.timeout, self.conf.download_mirrors
|
||||
)
|
||||
defer.returnValue(sd_blob)
|
||||
|
||||
@@ -162,7 +163,7 @@ class GetStream:
|
||||
def _download(self, sd_blob, name, key_fee, txid, nout, file_name=None):
|
||||
self.downloader = yield self._create_downloader(sd_blob, file_name=file_name)
|
||||
yield self.pay_key_fee(key_fee, name)
|
||||
yield self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout))
|
||||
yield f2d(self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout)))
|
||||
self.finished_deferred = self.downloader.start()
|
||||
self.downloader.download_manager.progress_manager.wrote_first_data.addCallback(
|
||||
self.data_downloading_deferred.callback
|
||||
|
||||
@@ -2,27 +2,30 @@ import binascii
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer, task
|
||||
from lbrynet import utils, conf
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DHTHashAnnouncer:
|
||||
def __init__(self, dht_node, storage, concurrent_announcers=None):
|
||||
def __init__(self, conf: Config, dht_node, storage, concurrent_announcers=None):
|
||||
self.conf = conf
|
||||
self.dht_node = dht_node
|
||||
self.storage = storage
|
||||
self.clock = dht_node.clock
|
||||
self.peer_port = dht_node.peerPort
|
||||
self.hash_queue = []
|
||||
if concurrent_announcers is None:
|
||||
self.concurrent_announcers = conf.settings['concurrent_announcers']
|
||||
self.concurrent_announcers = conf.concurrent_announcers
|
||||
else:
|
||||
self.concurrent_announcers = concurrent_announcers
|
||||
self._manage_lc = None
|
||||
if self.concurrent_announcers:
|
||||
self._manage_lc = task.LoopingCall(self.manage)
|
||||
self._manage_lc.clock = self.clock
|
||||
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.settings['concurrent_announcers'] or 1)
|
||||
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.concurrent_announcers or 1)
|
||||
|
||||
def start(self):
|
||||
if self._manage_lc:
|
||||
@@ -38,7 +41,7 @@ class DHTHashAnnouncer:
|
||||
now = self.clock.seconds()
|
||||
if storing_node_ids:
|
||||
result = (now, storing_node_ids)
|
||||
yield self.storage.update_last_announced_blob(blob_hash, now)
|
||||
yield f2d(self.storage.update_last_announced_blob(blob_hash, now))
|
||||
log.debug("Stored %s to %i peers", blob_hash[:16], len(storing_node_ids))
|
||||
else:
|
||||
result = (None, [])
|
||||
@@ -82,7 +85,7 @@ class DHTHashAnnouncer:
|
||||
if not self.dht_node.contacts:
|
||||
log.info("Not ready to start announcing hashes")
|
||||
return
|
||||
need_reannouncement = yield self.storage.get_blobs_to_announce()
|
||||
need_reannouncement = yield f2d(self.storage.get_blobs_to_announce())
|
||||
if need_reannouncement:
|
||||
yield self.immediate_announce(need_reannouncement)
|
||||
else:
|
||||
|
||||
@@ -2,7 +2,6 @@ import binascii
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
from lbrynet import conf
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -59,7 +58,7 @@ class DHTPeerFinder(DummyPeerFinder):
|
||||
def _execute_peer_search(self, dht_node, blob_hash, timeout):
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
finished_deferred = dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
|
||||
timeout = timeout or conf.settings['peer_search_timeout']
|
||||
timeout = timeout or self.component_manager.conf.peer_search_timeout
|
||||
if timeout:
|
||||
finished_deferred.addTimeout(timeout, dht_node.clock)
|
||||
try:
|
||||
|
||||
@@ -3,7 +3,7 @@ import logging
|
||||
import os
|
||||
|
||||
from lbrynet.blob.EncryptedFileCreator import create_lbry_file
|
||||
from lbrynet.extras.daemon.mime_types import guess_mime_type
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -43,25 +43,25 @@ class Publisher:
|
||||
claim_dict['stream']['source'] = {}
|
||||
claim_dict['stream']['source']['source'] = self.lbry_file.sd_hash
|
||||
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
|
||||
claim_dict['stream']['source']['contentType'] = guess_mime_type(file_path)
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
|
||||
tx = await self.wallet.claim_name(
|
||||
self.account, name, bid, claim_dict, self.certificate, holding_address
|
||||
)
|
||||
|
||||
# check if we have a file already for this claim (if this is a publish update with a new stream)
|
||||
old_stream_hashes = await d2f(self.storage.get_old_stream_hashes_for_claim_id(
|
||||
old_stream_hashes = await self.storage.get_old_stream_hashes_for_claim_id(
|
||||
tx.outputs[0].claim_id, self.lbry_file.stream_hash
|
||||
))
|
||||
)
|
||||
if old_stream_hashes:
|
||||
for lbry_file in filter(lambda l: l.stream_hash in old_stream_hashes,
|
||||
list(self.lbry_file_manager.lbry_files)):
|
||||
await d2f(self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False))
|
||||
log.info("Removed old stream for claim update: %s", lbry_file.stream_hash)
|
||||
|
||||
await d2f(self.storage.save_content_claim(
|
||||
await self.storage.save_content_claim(
|
||||
self.lbry_file.stream_hash, tx.outputs[0].id
|
||||
))
|
||||
)
|
||||
return tx
|
||||
|
||||
async def publish_stream(self, name, bid, claim_dict, stream_hash, holding_address=None):
|
||||
@@ -70,8 +70,8 @@ class Publisher:
|
||||
self.account, name, bid, claim_dict, self.certificate, holding_address
|
||||
)
|
||||
if stream_hash: # the stream_hash returned from the db will be None if this isn't a stream we have
|
||||
await d2f(self.storage.save_content_claim(
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
))
|
||||
)
|
||||
self.lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == stream_hash][0]
|
||||
return tx
|
||||
@@ -1,11 +1,12 @@
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer, task
|
||||
import aiohttp
|
||||
|
||||
from lbrynet import conf, utils
|
||||
from lbrynet.extras import looping_call_manager, system_info
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
|
||||
from lbrynet.extras import system_info
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
@@ -27,36 +28,82 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager:
|
||||
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
|
||||
self.analytics_api = analytics_api
|
||||
|
||||
def __init__(self, conf: Config, installation_id: str, session_id: str):
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._enabled = conf.share_usage_data
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.looping_call_manager = self._setup_looping_calls()
|
||||
self.context = context or self._make_context(
|
||||
system_info.get_platform(), conf.settings['wallet'])
|
||||
self.installation_id = installation_id or conf.settings.installation_id
|
||||
self.session_id = session_id or conf.settings.get_session_id()
|
||||
self.is_started = False
|
||||
self.context = self._make_context(system_info.get_platform(), 'torba')
|
||||
self.installation_id = installation_id
|
||||
self.session_id = session_id
|
||||
self.task: asyncio.Task = None
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
api = Api.new_instance(enabled)
|
||||
return cls(api)
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
# Things We Track
|
||||
def send_new_download_start(self, download_id, name, claim_dict):
|
||||
self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
def start(self):
|
||||
if self._enabled and self.task is None:
|
||||
self.task = asyncio.create_task(self.run())
|
||||
log.info("Start")
|
||||
|
||||
def send_new_download_success(self, download_id, name, claim_dict):
|
||||
self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
async def run(self):
|
||||
while True:
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
|
||||
def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
self.task.cancel()
|
||||
|
||||
async def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
request_kwargs = {
|
||||
'method': 'POST',
|
||||
'url': self.url + endpoint,
|
||||
'headers': {'Connection': 'Close'},
|
||||
'auth': aiohttp.BasicAuth(self._write_key, ''),
|
||||
'json': data,
|
||||
'cookies': self.cookies
|
||||
}
|
||||
try:
|
||||
async with aiohttp.request(**request_kwargs) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except Exception as e:
|
||||
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
|
||||
|
||||
async def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if self._enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post('/track', event)
|
||||
|
||||
async def send_new_download_start(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_success(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
|
||||
'message': str(e),
|
||||
})
|
||||
|
||||
def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
self.analytics_api.track({
|
||||
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
await self.track({
|
||||
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
|
||||
'event': NEW_DOWNLOAD_STAT,
|
||||
'properties': self._event_properties({
|
||||
@@ -70,90 +117,53 @@ class Manager:
|
||||
'timestamp': utils.isonow(),
|
||||
})
|
||||
|
||||
def send_upnp_setup_success_fail(self, success, status):
|
||||
self.analytics_api.track(
|
||||
async def send_upnp_setup_success_fail(self, success, status):
|
||||
await self.track(
|
||||
self._event(UPNP_SETUP, {
|
||||
'success': success,
|
||||
'status': status,
|
||||
})
|
||||
)
|
||||
|
||||
def send_server_startup(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP))
|
||||
async def send_server_startup(self):
|
||||
await self.track(self._event(SERVER_STARTUP))
|
||||
|
||||
def send_server_startup_success(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
async def send_server_startup_success(self):
|
||||
await self.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
|
||||
def send_server_startup_error(self, message):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
async def send_server_startup_error(self, message):
|
||||
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
def send_download_started(self, id_, name, claim_dict=None):
|
||||
self.analytics_api.track(
|
||||
async def send_download_started(self, id_, name, claim_dict=None):
|
||||
await self.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
async def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
async def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
def send_claim_action(self, action):
|
||||
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
async def send_claim_action(self, action):
|
||||
await self.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
def send_new_channel(self):
|
||||
self.analytics_api.track(self._event(NEW_CHANNEL))
|
||||
async def send_new_channel(self):
|
||||
await self.track(self._event(NEW_CHANNEL))
|
||||
|
||||
def send_credits_sent(self):
|
||||
self.analytics_api.track(self._event(CREDITS_SENT))
|
||||
async def send_credits_sent(self):
|
||||
await self.track(self._event(CREDITS_SENT))
|
||||
|
||||
def _send_heartbeat(self):
|
||||
self.analytics_api.track(self._event(HEARTBEAT))
|
||||
async def _send_heartbeat(self):
|
||||
await self.track(self._event(HEARTBEAT))
|
||||
|
||||
def _update_tracked_metrics(self):
|
||||
async def _update_tracked_metrics(self):
|
||||
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
# Setup / Shutdown
|
||||
|
||||
def start(self):
|
||||
if not self.is_started:
|
||||
for name, _, interval in self._get_looping_calls():
|
||||
self.looping_call_manager.start(name, interval)
|
||||
self.is_started = True
|
||||
|
||||
def shutdown(self):
|
||||
self.looping_call_manager.shutdown()
|
||||
|
||||
def register_repeating_metric(self, event_name, value_generator, frequency=300):
|
||||
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
|
||||
self.looping_call_manager.register_looping_call(event_name, lcall)
|
||||
lcall.start(frequency)
|
||||
|
||||
def _get_looping_calls(self):
|
||||
return [
|
||||
('send_heartbeat', self._send_heartbeat, 300),
|
||||
('update_tracked_metrics', self._update_tracked_metrics, 600),
|
||||
]
|
||||
|
||||
def _setup_looping_calls(self):
|
||||
call_manager = looping_call_manager.LoopingCallManager()
|
||||
for name, fn, _ in self._get_looping_calls():
|
||||
call_manager.register_looping_call(name, task.LoopingCall(fn))
|
||||
return call_manager
|
||||
|
||||
def _send_repeating_metric(self, event_name, value_generator):
|
||||
result = value_generator()
|
||||
self._if_deferred(result, self._send_repeating_metric_value, event_name)
|
||||
|
||||
def _send_repeating_metric_value(self, result, event_name):
|
||||
should_send, value = result
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(event_name, value))
|
||||
await self.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self._tracked_data[metric].append(value)
|
||||
@@ -238,65 +248,3 @@ class Manager:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
|
||||
if isinstance(maybe_deferred, defer.Deferred):
|
||||
maybe_deferred.addCallback(callback, *args, **kwargs)
|
||||
else:
|
||||
callback(maybe_deferred, *args, **kwargs)
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self, cookies, url, write_key, enabled):
|
||||
self.cookies = cookies
|
||||
self.url = url
|
||||
self._write_key = write_key
|
||||
self._enabled = enabled
|
||||
|
||||
def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
def update_cookies(response):
|
||||
self.cookies.update(response.cookies())
|
||||
return response
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
headers = {b"Connection": b"close"}
|
||||
d = treq.post(self.url + endpoint, auth=(self._write_key, ''), json=data,
|
||||
headers=headers, cookies=self.cookies)
|
||||
d.addCallback(update_cookies)
|
||||
return d
|
||||
|
||||
def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if not self._enabled:
|
||||
return defer.succeed('Analytics disabled')
|
||||
|
||||
def _log_error(failure, event):
|
||||
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
|
||||
|
||||
log.debug('Sending track event: %s', event)
|
||||
d = self._post('/track', event)
|
||||
d.addErrback(_log_error, event)
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
"""Initialize an instance using values from the configuration"""
|
||||
if enabled is None:
|
||||
enabled = conf.settings['share_usage_data']
|
||||
return cls(
|
||||
{},
|
||||
conf.settings['ANALYTICS_ENDPOINT'],
|
||||
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
|
||||
enabled,
|
||||
)
|
||||
Whitespace-only changes.
@@ -1,34 +0,0 @@
|
||||
import logging
|
||||
from zope.interface import implementer
|
||||
from twisted.cred import portal, checkers, credentials, error as cred_error
|
||||
from twisted.internet import defer
|
||||
from twisted.web import resource
|
||||
from lbrynet.extras.daemon.auth.keyring import Keyring
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@implementer(portal.IRealm)
|
||||
class HttpPasswordRealm:
|
||||
def __init__(self, resource):
|
||||
self.resource = resource
|
||||
|
||||
def requestAvatar(self, avatarId, mind, *interfaces):
|
||||
log.debug("Processing request for %s", avatarId)
|
||||
if resource.IResource in interfaces:
|
||||
return (resource.IResource, self.resource, lambda: None)
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
@implementer(checkers.ICredentialsChecker)
|
||||
class PasswordChecker:
|
||||
credentialInterfaces = (credentials.IUsernamePassword,)
|
||||
|
||||
def __init__(self, keyring: Keyring):
|
||||
self.api_key = keyring.api_key
|
||||
|
||||
def requestAvatarId(self, creds):
|
||||
if creds.checkPassword(self.api_key.secret.encode()) and creds.username == self.api_key.name.encode():
|
||||
return defer.succeed(creds.username)
|
||||
log.warning('Incorrect username or password')
|
||||
return defer.fail(cred_error.UnauthorizedLogin('Incorrect username or password'))
|
||||
@@ -1,130 +0,0 @@
|
||||
import json
|
||||
import aiohttp
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.daemon.auth.keyring import Keyring, APIKey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
USER_AGENT = "AuthServiceProxy/0.1"
|
||||
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
|
||||
TWISTED_SESSION = "TWISTED_SESSION"
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
HTTP_TIMEOUT = 30
|
||||
|
||||
|
||||
class JSONRPCException(Exception):
|
||||
def __init__(self, rpc_error):
|
||||
super().__init__()
|
||||
self.error = rpc_error
|
||||
|
||||
|
||||
class UnAuthAPIClient:
|
||||
def __init__(self, host, port, session):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.session = session
|
||||
|
||||
def __getattr__(self, method):
|
||||
async def f(*args, **kwargs):
|
||||
return await self.call(method, [args, kwargs])
|
||||
|
||||
return f
|
||||
|
||||
@classmethod
|
||||
async def from_url(cls, url):
|
||||
url_fragment = urlparse(url)
|
||||
host = url_fragment.hostname
|
||||
port = url_fragment.port
|
||||
connector = aiohttp.TCPConnector(
|
||||
ssl=None if not conf.settings['use_https'] else Keyring.load_from_disk().ssl_context
|
||||
)
|
||||
session = aiohttp.ClientSession(connector=connector)
|
||||
return cls(host, port, session)
|
||||
|
||||
async def call(self, method, params=None):
|
||||
message = {'method': method, 'params': params}
|
||||
async with self.session.get(conf.settings.get_api_connection_string(), json=message) as resp:
|
||||
return await resp.json()
|
||||
|
||||
|
||||
class AuthAPIClient:
|
||||
def __init__(self, key, session, cookies, url, login_url):
|
||||
self.session = session
|
||||
self.__api_key = key
|
||||
self.__login_url = login_url
|
||||
self.__id_count = 0
|
||||
self.__url = url
|
||||
self.__cookies = cookies
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name.startswith('__') and name.endswith('__'):
|
||||
raise AttributeError(name)
|
||||
|
||||
def f(*args, **kwargs):
|
||||
return self.call(name, [args, kwargs])
|
||||
|
||||
return f
|
||||
|
||||
async def call(self, method, params=None):
|
||||
params = params or {}
|
||||
self.__id_count += 1
|
||||
|
||||
pre_auth_post_data = {
|
||||
'version': '2',
|
||||
'method': method,
|
||||
'params': params,
|
||||
'id': self.__id_count
|
||||
}
|
||||
to_auth = json.dumps(pre_auth_post_data, sort_keys=True)
|
||||
auth_msg = self.__api_key.get_hmac(to_auth).decode()
|
||||
pre_auth_post_data.update({'hmac': auth_msg})
|
||||
post_data = json.dumps(pre_auth_post_data)
|
||||
|
||||
headers = {
|
||||
'Host': self.__url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'
|
||||
}
|
||||
|
||||
async with self.session.post(self.__login_url, data=post_data, headers=headers) as resp:
|
||||
if resp is None:
|
||||
raise JSONRPCException({'code': -342, 'message': 'missing HTTP response from server'})
|
||||
resp.raise_for_status()
|
||||
|
||||
next_secret = resp.headers.get(LBRY_SECRET, False)
|
||||
if next_secret:
|
||||
self.__api_key.secret = next_secret
|
||||
|
||||
return await resp.json()
|
||||
|
||||
@classmethod
|
||||
async def get_client(cls, key_name=None):
|
||||
api_key_name = key_name or "api"
|
||||
keyring = Keyring.load_from_disk()
|
||||
|
||||
api_key = keyring.api_key
|
||||
login_url = conf.settings.get_api_connection_string(api_key_name, api_key.secret)
|
||||
url = urlparse(login_url)
|
||||
|
||||
headers = {
|
||||
'Host': url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'
|
||||
}
|
||||
connector = aiohttp.TCPConnector(ssl=None if not conf.settings['use_https'] else keyring.ssl_context)
|
||||
session = aiohttp.ClientSession(connector=connector)
|
||||
|
||||
async with session.post(login_url, headers=headers) as r:
|
||||
cookies = r.cookies
|
||||
uid = cookies.get(TWISTED_SECURE_SESSION if conf.settings['use_https'] else TWISTED_SESSION).value
|
||||
api_key = APIKey.create(seed=uid.encode())
|
||||
return cls(api_key, session, cookies, url, login_url)
|
||||
|
||||
|
||||
class LBRYAPIClient:
|
||||
@staticmethod
|
||||
def get_client():
|
||||
return AuthAPIClient.get_client() if conf.settings['use_auth_http'] else \
|
||||
UnAuthAPIClient.from_url(conf.settings.get_api_connection_string())
|
||||
@@ -1,50 +0,0 @@
|
||||
import logging
|
||||
|
||||
from twisted.web import server, guard, resource
|
||||
from twisted.cred import portal
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.daemon.auth.auth import PasswordChecker, HttpPasswordRealm
|
||||
from lbrynet.extras.daemon.auth.keyring import Keyring
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HTTPJSONRPCFactory(server.Site):
|
||||
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
|
||||
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
|
||||
self.use_ssl = False
|
||||
|
||||
|
||||
class HTTPSJSONRPCFactory(server.Site):
|
||||
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
|
||||
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
|
||||
self.options = keyring.private_certificate.options()
|
||||
self.use_ssl = True
|
||||
|
||||
|
||||
class AuthJSONRPCResource(resource.Resource):
|
||||
def __init__(self, protocol):
|
||||
resource.Resource.__init__(self)
|
||||
self.putChild(b"", protocol)
|
||||
self.putChild(conf.settings['API_ADDRESS'].encode(), protocol)
|
||||
|
||||
def getChild(self, name, request):
|
||||
request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
|
||||
request.setHeader('expires', '0')
|
||||
return self if name == '' else resource.Resource.getChild(self, name, request)
|
||||
|
||||
def getServerFactory(self, keyring: Keyring, use_authentication: bool, use_https: bool) -> server.Site:
|
||||
factory_class = HTTPSJSONRPCFactory if use_https else HTTPJSONRPCFactory
|
||||
if use_authentication:
|
||||
log.info("Using authenticated API")
|
||||
checker = PasswordChecker(keyring)
|
||||
realm = HttpPasswordRealm(self)
|
||||
portal_to_realm = portal.Portal(realm, [checker, ])
|
||||
root = guard.HTTPAuthSessionWrapper(
|
||||
portal_to_realm, [guard.BasicCredentialFactory(b'Login to lbrynet api'), ]
|
||||
)
|
||||
else:
|
||||
log.info("Using non-authenticated API")
|
||||
root = self
|
||||
return factory_class(root, keyring)
|
||||
@@ -1,131 +0,0 @@
|
||||
import os
|
||||
import datetime
|
||||
import hmac
|
||||
import hashlib
|
||||
import base58
|
||||
import logging
|
||||
from OpenSSL.crypto import FILETYPE_PEM
|
||||
from ssl import create_default_context, SSLContext
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.x509.name import NameOID, NameAttribute
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from twisted.internet import ssl
|
||||
|
||||
from lbrynet import conf
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def sha(x: bytes) -> str:
|
||||
h = hashlib.sha256(x).digest()
|
||||
return base58.b58encode(h).decode()
|
||||
|
||||
|
||||
def generate_key(x: bytes = None) -> str:
|
||||
if not x:
|
||||
return sha(os.urandom(256))
|
||||
else:
|
||||
return sha(x)
|
||||
|
||||
|
||||
class APIKey:
|
||||
def __init__(self, secret: str, name: str):
|
||||
self.secret = secret
|
||||
self.name = name
|
||||
|
||||
@classmethod
|
||||
def create(cls, seed=None, name=None):
|
||||
secret = generate_key(seed)
|
||||
return APIKey(secret, name)
|
||||
|
||||
def _raw_key(self) -> str:
|
||||
return base58.b58decode(self.secret)
|
||||
|
||||
def get_hmac(self, message) -> str:
|
||||
decoded_key = self._raw_key()
|
||||
signature = hmac.new(decoded_key, message.encode(), hashlib.sha256)
|
||||
return base58.b58encode(signature.digest())
|
||||
|
||||
def compare_hmac(self, message, token) -> bool:
|
||||
decoded_token = base58.b58decode(token)
|
||||
target = base58.b58decode(self.get_hmac(message))
|
||||
|
||||
try:
|
||||
if len(decoded_token) != len(target):
|
||||
return False
|
||||
return hmac.compare_digest(decoded_token, target)
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
class Keyring:
|
||||
encoding = serialization.Encoding.PEM
|
||||
filetype = FILETYPE_PEM
|
||||
|
||||
def __init__(self, api_key: APIKey, public_certificate: str, private_certificate: ssl.PrivateCertificate = None):
|
||||
self.api_key: APIKey = api_key
|
||||
self.public_certificate: str = public_certificate
|
||||
self.private_certificate: (ssl.PrivateCertificate or None) = private_certificate
|
||||
self.ssl_context: SSLContext = create_default_context(cadata=self.public_certificate)
|
||||
|
||||
@classmethod
|
||||
def load_from_disk(cls):
|
||||
api_key_path = os.path.join(conf.settings.data_dir, 'auth_token')
|
||||
api_ssl_cert_path = os.path.join(conf.settings.data_dir, 'api_ssl_cert.pem')
|
||||
if not os.path.isfile(api_key_path) or not os.path.isfile(api_ssl_cert_path):
|
||||
return
|
||||
with open(api_key_path, 'rb') as f:
|
||||
api_key = APIKey(f.read().decode(), "api")
|
||||
with open(api_ssl_cert_path, 'rb') as f:
|
||||
public_cert = f.read().decode()
|
||||
return cls(api_key, public_cert)
|
||||
|
||||
@classmethod
|
||||
def generate_and_save(cls):
|
||||
dns = conf.settings['api_host']
|
||||
private_key = rsa.generate_private_key(
|
||||
public_exponent=65537,
|
||||
key_size=4096,
|
||||
backend=default_backend()
|
||||
)
|
||||
subject = issuer = x509.Name([
|
||||
NameAttribute(NameOID.COUNTRY_NAME, "US"),
|
||||
NameAttribute(NameOID.ORGANIZATION_NAME, "LBRY"),
|
||||
NameAttribute(NameOID.COMMON_NAME, "LBRY API"),
|
||||
])
|
||||
alternative_name = x509.SubjectAlternativeName([x509.DNSName(dns)])
|
||||
certificate = x509.CertificateBuilder(
|
||||
subject_name=subject,
|
||||
issuer_name=issuer,
|
||||
public_key=private_key.public_key(),
|
||||
serial_number=x509.random_serial_number(),
|
||||
not_valid_before=datetime.datetime.utcnow(),
|
||||
not_valid_after=datetime.datetime.utcnow() + datetime.timedelta(days=365),
|
||||
extensions=[x509.Extension(oid=alternative_name.oid, critical=False, value=alternative_name)]
|
||||
).sign(private_key, hashes.SHA256(), default_backend())
|
||||
public_certificate = certificate.public_bytes(cls.encoding).decode()
|
||||
private_certificate = ssl.PrivateCertificate.load(
|
||||
public_certificate,
|
||||
ssl.KeyPair.load(
|
||||
private_key.private_bytes(
|
||||
encoding=cls.encoding,
|
||||
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
encryption_algorithm=serialization.NoEncryption()
|
||||
).decode(),
|
||||
cls.filetype
|
||||
),
|
||||
cls.filetype
|
||||
)
|
||||
|
||||
auth_token = APIKey.create(seed=None, name="api")
|
||||
|
||||
with open(os.path.join(conf.settings.data_dir, 'auth_token'), 'wb') as f:
|
||||
f.write(auth_token.secret.encode())
|
||||
|
||||
with open(os.path.join(conf.settings.data_dir, 'api_ssl_cert.pem'), 'wb') as f:
|
||||
f.write(public_certificate.encode())
|
||||
|
||||
return cls(auth_token, public_certificate, private_certificate)
|
||||
@@ -1,611 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from urllib import parse as urlparse
|
||||
import json
|
||||
import inspect
|
||||
import signal
|
||||
from functools import wraps
|
||||
from twisted.web import server
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet.error import ConnectionDone, ConnectionLost
|
||||
from txjsonrpc import jsonrpclib
|
||||
from traceback import format_exc
|
||||
|
||||
from lbrynet import conf, utils
|
||||
from lbrynet.extras.daemon import analytics
|
||||
from lbrynet.p2p.Error import InvalidAuthenticationToken
|
||||
from lbrynet.p2p.Error import ComponentsNotStarted, ComponentStartConditionNotMet
|
||||
from lbrynet.extras.looping_call_manager import LoopingCallManager
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.auth.keyring import APIKey, Keyring
|
||||
from lbrynet.extras.daemon.auth.undecorated import undecorated
|
||||
from lbrynet.extras.daemon.auth.factory import AuthJSONRPCResource
|
||||
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
EMPTY_PARAMS = [{}]
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
|
||||
|
||||
class JSONRPCError:
|
||||
# http://www.jsonrpc.org/specification#error_object
|
||||
CODE_PARSE_ERROR = -32700 # Invalid JSON. Error while parsing the JSON text.
|
||||
CODE_INVALID_REQUEST = -32600 # The JSON sent is not a valid Request object.
|
||||
CODE_METHOD_NOT_FOUND = -32601 # The method does not exist / is not available.
|
||||
CODE_INVALID_PARAMS = -32602 # Invalid method parameter(s).
|
||||
CODE_INTERNAL_ERROR = -32603 # Internal JSON-RPC error (I think this is like a 500?)
|
||||
CODE_APPLICATION_ERROR = -32500 # Generic error with our app??
|
||||
CODE_AUTHENTICATION_ERROR = -32501 # Authentication failed
|
||||
|
||||
MESSAGES = {
|
||||
CODE_PARSE_ERROR: "Parse Error. Data is not valid JSON.",
|
||||
CODE_INVALID_REQUEST: "JSON data is not a valid Request",
|
||||
CODE_METHOD_NOT_FOUND: "Method Not Found",
|
||||
CODE_INVALID_PARAMS: "Invalid Params",
|
||||
CODE_INTERNAL_ERROR: "Internal Error",
|
||||
CODE_AUTHENTICATION_ERROR: "Authentication Failed",
|
||||
}
|
||||
|
||||
HTTP_CODES = {
|
||||
CODE_INVALID_REQUEST: 400,
|
||||
CODE_PARSE_ERROR: 400,
|
||||
CODE_INVALID_PARAMS: 400,
|
||||
CODE_METHOD_NOT_FOUND: 404,
|
||||
CODE_INTERNAL_ERROR: 500,
|
||||
CODE_APPLICATION_ERROR: 500,
|
||||
CODE_AUTHENTICATION_ERROR: 401,
|
||||
}
|
||||
|
||||
def __init__(self, message, code=CODE_APPLICATION_ERROR, traceback=None, data=None):
|
||||
assert isinstance(code, int), "'code' must be an int"
|
||||
assert (data is None or isinstance(data, dict)), "'data' must be None or a dict"
|
||||
self.code = code
|
||||
if message is None:
|
||||
message = self.MESSAGES[code] if code in self.MESSAGES else "API Error"
|
||||
self.message = message
|
||||
self.data = {} if data is None else data
|
||||
self.traceback = []
|
||||
if traceback is not None:
|
||||
trace_lines = traceback.split("\n")
|
||||
for i, t in enumerate(trace_lines):
|
||||
if "--- <exception caught here> ---" in t:
|
||||
if len(trace_lines) > i + 1:
|
||||
self.traceback = [j for j in trace_lines[i+1:] if j]
|
||||
break
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'code': self.code,
|
||||
'message': self.message,
|
||||
'data': self.traceback
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def create_from_exception(cls, message, code=CODE_APPLICATION_ERROR, traceback=None):
|
||||
return cls(message, code=code, traceback=traceback)
|
||||
|
||||
|
||||
class UnknownAPIMethodError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def trap(err, *to_trap):
|
||||
err.trap(*to_trap)
|
||||
|
||||
|
||||
def jsonrpc_dumps_pretty(obj, **kwargs):
|
||||
try:
|
||||
id_ = kwargs.pop("id")
|
||||
except KeyError:
|
||||
id_ = None
|
||||
|
||||
if isinstance(obj, JSONRPCError):
|
||||
data = {"jsonrpc": "2.0", "error": obj.to_dict(), "id": id_}
|
||||
else:
|
||||
data = {"jsonrpc": "2.0", "result": obj, "id": id_}
|
||||
|
||||
return json.dumps(data, cls=JSONResponseEncoder, sort_keys=True, indent=2, **kwargs) + "\n"
|
||||
|
||||
|
||||
class JSONRPCServerType(type):
|
||||
def __new__(mcs, name, bases, newattrs):
|
||||
klass = type.__new__(mcs, name, bases, newattrs)
|
||||
klass.callable_methods = {}
|
||||
klass.deprecated_methods = {}
|
||||
|
||||
for methodname in dir(klass):
|
||||
if methodname.startswith("jsonrpc_"):
|
||||
method = getattr(klass, methodname)
|
||||
if not hasattr(method, '_deprecated'):
|
||||
klass.callable_methods.update({methodname.split("jsonrpc_")[1]: method})
|
||||
else:
|
||||
klass.deprecated_methods.update({methodname.split("jsonrpc_")[1]: method})
|
||||
return klass
|
||||
|
||||
|
||||
class AuthorizedBase(metaclass=JSONRPCServerType):
|
||||
|
||||
@staticmethod
|
||||
def deprecated(new_command=None):
|
||||
def _deprecated_wrapper(f):
|
||||
f.new_command = new_command
|
||||
f._deprecated = True
|
||||
return f
|
||||
return _deprecated_wrapper
|
||||
|
||||
@staticmethod
|
||||
def requires(*components, **conditions):
|
||||
if conditions and ["conditions"] != list(conditions.keys()):
|
||||
raise SyntaxError("invalid conditions argument")
|
||||
condition_names = conditions.get("conditions", [])
|
||||
|
||||
def _wrap(fn):
|
||||
@wraps(fn)
|
||||
def _inner(*args, **kwargs):
|
||||
component_manager = args[0].component_manager
|
||||
for condition_name in condition_names:
|
||||
condition_result, err_msg = component_manager.evaluate_condition(condition_name)
|
||||
if not condition_result:
|
||||
raise ComponentStartConditionNotMet(err_msg)
|
||||
if not component_manager.all_components_running(*components):
|
||||
raise ComponentsNotStarted("the following required components have not yet started: "
|
||||
"%s" % json.dumps(components))
|
||||
return fn(*args, **kwargs)
|
||||
return _inner
|
||||
return _wrap
|
||||
|
||||
|
||||
class AuthJSONRPCServer(AuthorizedBase):
|
||||
"""
|
||||
Authorized JSONRPC server used as the base class for the LBRY API
|
||||
|
||||
API methods are named with a leading "jsonrpc_"
|
||||
|
||||
Attributes:
|
||||
sessions (dict): (dict): {<session id>: <lbrynet.daemon.auth.util.APIKey>}
|
||||
callable_methods (dict): {<api method name>: <api method>}
|
||||
|
||||
Authentication:
|
||||
If use_authentication is true, basic HTTP and HMAC authentication will be used for all requests and the
|
||||
service url will require a username and password.
|
||||
|
||||
To start an authenticated session a client sends an HTTP POST to <user>:<password>@<api host>:<api port>.
|
||||
If accepted, the server replies with a TWISTED_SESSION cookie containing a session id and the message "OK".
|
||||
The client initializes their shared secret for hmac to be the b64 encoded sha256 of their session id.
|
||||
|
||||
To send an authenticated request a client sends an HTTP POST to the auth api url with the TWISTED_SESSION
|
||||
cookie and includes a hmac token in the message using the previously set shared secret. If the token is valid
|
||||
the server will randomize the shared secret and return the new value under the LBRY_SECRET header, which the
|
||||
client uses to generate the token for their next request.
|
||||
"""
|
||||
#implements(resource.IResource)
|
||||
|
||||
isLeaf = True
|
||||
allowed_during_startup = []
|
||||
component_attributes = {}
|
||||
|
||||
def __init__(self, analytics_manager=None, component_manager=None, use_authentication=None, use_https=None,
|
||||
to_skip=None, looping_calls=None, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.analytics_manager = analytics_manager or analytics.Manager.new_instance()
|
||||
self.component_manager = component_manager or ComponentManager(
|
||||
analytics_manager=self.analytics_manager,
|
||||
skip_components=to_skip or [],
|
||||
reactor=reactor
|
||||
)
|
||||
self.looping_call_manager = LoopingCallManager({n: lc for n, (lc, t) in (looping_calls or {}).items()})
|
||||
self._looping_call_times = {n: t for n, (lc, t) in (looping_calls or {}).items()}
|
||||
self._use_authentication = use_authentication or conf.settings['use_auth_http']
|
||||
self._use_https = use_https or conf.settings['use_https']
|
||||
self.listening_port = None
|
||||
self._component_setup_deferred = None
|
||||
self.announced_startup = False
|
||||
self.sessions = {}
|
||||
self.server = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_listening(self):
|
||||
from twisted.internet import reactor, error as tx_error
|
||||
|
||||
try:
|
||||
self.server = self.get_server_factory()
|
||||
if self.server.use_ssl:
|
||||
log.info("Using SSL")
|
||||
self.listening_port = reactor.listenSSL(
|
||||
conf.settings['api_port'], self.server, self.server.options, interface=conf.settings['api_host']
|
||||
)
|
||||
else:
|
||||
log.info("Not using SSL")
|
||||
self.listening_port = reactor.listenTCP(
|
||||
conf.settings['api_port'], self.server, interface=conf.settings['api_host']
|
||||
)
|
||||
log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
|
||||
yield self.setup()
|
||||
self.analytics_manager.send_server_startup_success()
|
||||
except tx_error.CannotListenError:
|
||||
log.error('lbrynet API failed to bind TCP %s:%i for listening. Daemon is already running or this port is '
|
||||
'already in use by another application.', conf.settings['api_host'], conf.settings['api_port'])
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
except defer.CancelledError:
|
||||
log.info("shutting down before finished starting")
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
except Exception as err:
|
||||
self.analytics_manager.send_server_startup_error(str(err))
|
||||
log.exception('Failed to start lbrynet-daemon')
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
|
||||
def setup(self):
|
||||
from twisted.internet import reactor
|
||||
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
|
||||
if not self.analytics_manager.is_started:
|
||||
self.analytics_manager.start()
|
||||
self.analytics_manager.send_server_startup()
|
||||
for lc_name, lc_time in self._looping_call_times.items():
|
||||
self.looping_call_manager.start(lc_name, lc_time)
|
||||
|
||||
def update_attribute(setup_result, component):
|
||||
setattr(self, self.component_attributes[component.component_name], component.component)
|
||||
|
||||
kwargs = {component: update_attribute for component in self.component_attributes.keys()}
|
||||
self._component_setup_deferred = self.component_manager.setup(**kwargs)
|
||||
return self._component_setup_deferred
|
||||
|
||||
@staticmethod
|
||||
def _already_shutting_down(sig_num, frame):
|
||||
log.info("Already shutting down")
|
||||
|
||||
def _shutdown(self):
|
||||
# ignore INT/TERM signals once shutdown has started
|
||||
signal.signal(signal.SIGINT, self._already_shutting_down)
|
||||
signal.signal(signal.SIGTERM, self._already_shutting_down)
|
||||
if self.listening_port:
|
||||
self.listening_port.stopListening()
|
||||
self.looping_call_manager.shutdown()
|
||||
if self.server is not None:
|
||||
for session in list(self.server.sessions.values()):
|
||||
session.expire()
|
||||
if self.analytics_manager:
|
||||
self.analytics_manager.shutdown()
|
||||
try:
|
||||
self._component_setup_deferred.cancel()
|
||||
except (AttributeError, defer.CancelledError):
|
||||
pass
|
||||
if self.component_manager is not None:
|
||||
d = self.component_manager.stop()
|
||||
d.addErrback(log.fail(), 'Failure while shutting down')
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
return d
|
||||
|
||||
def get_server_factory(self):
|
||||
keyring = None
|
||||
if self._use_authentication or self._use_https:
|
||||
keyring = Keyring.generate_and_save()
|
||||
return AuthJSONRPCResource(self).getServerFactory(keyring, self._use_authentication, self._use_https)
|
||||
|
||||
def _set_headers(self, request, data, update_secret=False):
|
||||
if conf.settings['allowed_origin']:
|
||||
request.setHeader("Access-Control-Allow-Origin", conf.settings['allowed_origin'])
|
||||
request.setHeader("Content-Type", "application/json")
|
||||
request.setHeader("Accept", "application/json-rpc")
|
||||
request.setHeader("Content-Length", str(len(data)))
|
||||
if update_secret:
|
||||
session_id = request.getSession().uid
|
||||
request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret)
|
||||
|
||||
@staticmethod
|
||||
def _render_message(request, message: str):
|
||||
request.write(message.encode())
|
||||
request.finish()
|
||||
|
||||
def _render_error(self, failure, request, id_):
|
||||
if isinstance(failure, JSONRPCError):
|
||||
error = failure
|
||||
elif isinstance(failure, Failure):
|
||||
# maybe failure is JSONRPCError wrapped in a twisted Failure
|
||||
error = failure.check(JSONRPCError)
|
||||
if error is None:
|
||||
# maybe its a twisted Failure with another type of error
|
||||
if hasattr(failure.type, "code"):
|
||||
error_code = failure.type.code
|
||||
else:
|
||||
error_code = JSONRPCError.CODE_APPLICATION_ERROR
|
||||
error = JSONRPCError.create_from_exception(
|
||||
failure.getErrorMessage() or failure.type.__name__,
|
||||
code=error_code,
|
||||
traceback=failure.getTraceback()
|
||||
)
|
||||
if not failure.check(ComponentsNotStarted, ComponentStartConditionNotMet):
|
||||
log.warning("error processing api request: %s\ntraceback: %s", error.message,
|
||||
"\n".join(error.traceback))
|
||||
else:
|
||||
# last resort, just cast it as a string
|
||||
error = JSONRPCError(str(failure))
|
||||
|
||||
response_content = jsonrpc_dumps_pretty(error, id=id_, ledger=self.ledger)
|
||||
self._set_headers(request, response_content)
|
||||
request.setResponseCode(200)
|
||||
self._render_message(request, response_content)
|
||||
|
||||
@staticmethod
|
||||
def _handle_dropped_request(result, d, function_name):
|
||||
if not d.called:
|
||||
log.warning("Cancelling dropped api request %s", function_name)
|
||||
d.cancel()
|
||||
|
||||
def render(self, request):
|
||||
try:
|
||||
return self._render(request)
|
||||
except BaseException as e:
|
||||
log.error(e)
|
||||
error = JSONRPCError.create_from_exception(str(e), traceback=format_exc())
|
||||
self._render_error(error, request, None)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def _render(self, request):
|
||||
time_in = utils.now()
|
||||
# if not self._check_headers(request):
|
||||
# self._render_error(Failure(InvalidHeaderError()), request, None)
|
||||
# return server.NOT_DONE_YET
|
||||
session = request.getSession()
|
||||
session_id = session.uid
|
||||
finished_deferred = request.notifyFinish()
|
||||
|
||||
if self._use_authentication:
|
||||
# if this is a new session, send a new secret and set the expiration
|
||||
# otherwise, session.touch()
|
||||
if self._initialize_session(session_id):
|
||||
def expire_session():
|
||||
self._unregister_user_session(session_id)
|
||||
|
||||
session.notifyOnExpire(expire_session)
|
||||
message = "OK"
|
||||
request.setResponseCode(200)
|
||||
self._set_headers(request, message, True)
|
||||
self._render_message(request, message)
|
||||
return server.NOT_DONE_YET
|
||||
else:
|
||||
session.touch()
|
||||
|
||||
request.content.seek(0, 0)
|
||||
content = request.content.read().decode()
|
||||
try:
|
||||
parsed = jsonrpclib.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
log.warning("Unable to decode request json")
|
||||
self._render_error(JSONRPCError(None, code=JSONRPCError.CODE_PARSE_ERROR), request, None)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
request_id = None
|
||||
try:
|
||||
function_name = parsed.get('method')
|
||||
args = parsed.get('params', {})
|
||||
request_id = parsed.get('id', None)
|
||||
token = parsed.pop('hmac', None)
|
||||
except AttributeError as err:
|
||||
log.warning(err)
|
||||
self._render_error(
|
||||
JSONRPCError(None, code=JSONRPCError.CODE_INVALID_REQUEST), request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
reply_with_next_secret = False
|
||||
if self._use_authentication:
|
||||
try:
|
||||
self._verify_token(session_id, parsed, token)
|
||||
except InvalidAuthenticationToken as err:
|
||||
log.warning("API validation failed")
|
||||
self._render_error(
|
||||
JSONRPCError.create_from_exception(
|
||||
str(err),
|
||||
code=JSONRPCError.CODE_AUTHENTICATION_ERROR,
|
||||
traceback=format_exc()
|
||||
),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
request.addCookie("TWISTED_SESSION", session_id)
|
||||
self._update_session_secret(session_id)
|
||||
reply_with_next_secret = True
|
||||
|
||||
try:
|
||||
fn = self._get_jsonrpc_method(function_name)
|
||||
except UnknownAPIMethodError as err:
|
||||
log.warning('Failed to get function %s: %s', function_name, err)
|
||||
self._render_error(
|
||||
JSONRPCError(None, code=JSONRPCError.CODE_METHOD_NOT_FOUND),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
if args in (EMPTY_PARAMS, []):
|
||||
_args, _kwargs = (), {}
|
||||
elif isinstance(args, dict):
|
||||
_args, _kwargs = (), args
|
||||
elif len(args) == 1 and isinstance(args[0], dict):
|
||||
# TODO: this is for backwards compatibility. Remove this once API and UI are updated
|
||||
# TODO: also delete EMPTY_PARAMS then
|
||||
_args, _kwargs = (), args[0]
|
||||
elif len(args) == 2 and isinstance(args[0], list) and isinstance(args[1], dict):
|
||||
_args, _kwargs = args
|
||||
else:
|
||||
raise ValueError('invalid args format')
|
||||
|
||||
params_error, erroneous_params = self._check_params(fn, _args, _kwargs)
|
||||
if params_error is not None:
|
||||
params_error_message = '{} for {} command: {}'.format(
|
||||
params_error, function_name, ', '.join(erroneous_params)
|
||||
)
|
||||
log.warning(params_error_message)
|
||||
self._render_error(
|
||||
JSONRPCError(params_error_message, code=JSONRPCError.CODE_INVALID_PARAMS),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
try:
|
||||
result = fn(self, *_args, **_kwargs)
|
||||
if isinstance(result, Deferred):
|
||||
d = result
|
||||
elif isinstance(result, Failure):
|
||||
d = defer.fail(result)
|
||||
elif asyncio.iscoroutine(result):
|
||||
d = Deferred.fromFuture(asyncio.ensure_future(result))
|
||||
else:
|
||||
d = defer.succeed(result)
|
||||
except:
|
||||
d = defer.fail(Failure(captureVars=Deferred.debug))
|
||||
|
||||
# finished_deferred will callback when the request is finished
|
||||
# and errback if something went wrong. If the errback is
|
||||
# called, cancel the deferred stack. This is to prevent
|
||||
# request.finish() from being called on a closed request.
|
||||
finished_deferred.addErrback(self._handle_dropped_request, d, function_name)
|
||||
|
||||
d.addCallback(self._callback_render, request, request_id, reply_with_next_secret)
|
||||
d.addErrback(trap, ConnectionDone, ConnectionLost, defer.CancelledError)
|
||||
d.addErrback(self._render_error, request, request_id)
|
||||
d.addBoth(lambda _: log.debug("%s took %f",
|
||||
function_name,
|
||||
(utils.now() - time_in).total_seconds()))
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def _register_user_session(self, session_id):
|
||||
"""
|
||||
Add or update a HMAC secret for a session
|
||||
|
||||
@param session_id:
|
||||
@return: secret
|
||||
"""
|
||||
log.info("Started new api session")
|
||||
token = APIKey.create(seed=session_id)
|
||||
self.sessions.update({session_id: token})
|
||||
|
||||
def _unregister_user_session(self, session_id):
|
||||
log.info("Unregister API session")
|
||||
del self.sessions[session_id]
|
||||
|
||||
def _check_headers(self, request):
|
||||
return (
|
||||
self._check_header_source(request, 'Origin') and
|
||||
self._check_header_source(request, 'Referer')
|
||||
)
|
||||
|
||||
def _check_header_source(self, request, header):
|
||||
"""Check if the source of the request is allowed based on the header value."""
|
||||
source = request.getHeader(header)
|
||||
if not self._check_source_of_request(source):
|
||||
log.warning("Attempted api call from invalid %s: %s", header, source)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _check_source_of_request(self, source):
|
||||
if source is None:
|
||||
return True
|
||||
if conf.settings['api_host'] == '0.0.0.0':
|
||||
return True
|
||||
server, port = self.get_server_port(source)
|
||||
return self._check_server_port(server, port)
|
||||
|
||||
def _check_server_port(self, server, port):
|
||||
api = (conf.settings['api_host'], conf.settings['api_port'])
|
||||
return (server, port) == api or self._is_from_allowed_origin(server, port)
|
||||
|
||||
def _is_from_allowed_origin(self, server, port):
|
||||
allowed_origin = conf.settings['allowed_origin']
|
||||
if not allowed_origin:
|
||||
return False
|
||||
if allowed_origin == '*':
|
||||
return True
|
||||
allowed_server, allowed_port = self.get_server_port(allowed_origin)
|
||||
return (allowed_server, allowed_port) == (server, port)
|
||||
|
||||
def get_server_port(self, origin):
|
||||
parsed = urlparse.urlparse(origin)
|
||||
server_port = parsed.netloc.split(':')
|
||||
assert len(server_port) <= 2
|
||||
if len(server_port) == 2:
|
||||
return server_port[0], int(server_port[1])
|
||||
else:
|
||||
return server_port[0], 80
|
||||
|
||||
def _verify_method_is_callable(self, function_path):
|
||||
if function_path not in self.callable_methods:
|
||||
raise UnknownAPIMethodError(function_path)
|
||||
|
||||
def _get_jsonrpc_method(self, function_path):
|
||||
if function_path in self.deprecated_methods:
|
||||
new_command = self.deprecated_methods[function_path].new_command
|
||||
log.warning('API function \"%s\" is deprecated, please update to use \"%s\"',
|
||||
function_path, new_command)
|
||||
function_path = new_command
|
||||
self._verify_method_is_callable(function_path)
|
||||
return self.callable_methods.get(function_path)
|
||||
|
||||
@staticmethod
|
||||
def _check_params(function, args_tup, args_dict):
|
||||
argspec = inspect.getfullargspec(undecorated(function))
|
||||
num_optional_params = 0 if argspec.defaults is None else len(argspec.defaults)
|
||||
|
||||
duplicate_params = [
|
||||
duplicate_param
|
||||
for duplicate_param in argspec.args[1:len(args_tup) + 1]
|
||||
if duplicate_param in args_dict
|
||||
]
|
||||
|
||||
if duplicate_params:
|
||||
return 'Duplicate parameters', duplicate_params
|
||||
|
||||
missing_required_params = [
|
||||
required_param
|
||||
for required_param in argspec.args[len(args_tup)+1:-num_optional_params]
|
||||
if required_param not in args_dict
|
||||
]
|
||||
if len(missing_required_params):
|
||||
return 'Missing required parameters', missing_required_params
|
||||
|
||||
extraneous_params = [] if argspec.varkw is not None else [
|
||||
extra_param
|
||||
for extra_param in args_dict
|
||||
if extra_param not in argspec.args[1:]
|
||||
]
|
||||
if len(extraneous_params):
|
||||
return 'Extraneous parameters', extraneous_params
|
||||
|
||||
return None, None
|
||||
|
||||
def _initialize_session(self, session_id):
|
||||
if not self.sessions.get(session_id):
|
||||
self._register_user_session(session_id)
|
||||
return True
|
||||
return False
|
||||
|
||||
def _verify_token(self, session_id, message, token):
|
||||
if token is None:
|
||||
raise InvalidAuthenticationToken('Authentication token not found')
|
||||
to_auth = json.dumps(message, sort_keys=True)
|
||||
api_key = self.sessions.get(session_id)
|
||||
if not api_key.compare_hmac(to_auth, token):
|
||||
raise InvalidAuthenticationToken('Invalid authentication token')
|
||||
|
||||
def _update_session_secret(self, session_id):
|
||||
self.sessions.update({session_id: APIKey.create(name=session_id)})
|
||||
|
||||
def _callback_render(self, result, request, id_, auth_required=False):
|
||||
try:
|
||||
message = jsonrpc_dumps_pretty(result, id=id_, ledger=self.ledger)
|
||||
request.setResponseCode(200)
|
||||
self._set_headers(request, message, auth_required)
|
||||
self._render_message(request, message)
|
||||
except Exception as err:
|
||||
log.exception("Failed to render API response: %s", result)
|
||||
self._render_error(err, request, id_)
|
||||
|
||||
@staticmethod
|
||||
def _render_response(result):
|
||||
return defer.succeed(result)
|
||||
+27
-28
@@ -1,11 +1,9 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import time
|
||||
import logging
|
||||
import json
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
from lbrynet.p2p.Error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -38,14 +36,14 @@ class MarketFeed:
|
||||
REQUESTS_TIMEOUT = 20
|
||||
EXCHANGE_RATE_UPDATE_RATE_SEC = 300
|
||||
|
||||
def __init__(self, market, name, url, params, fee):
|
||||
def __init__(self, market: str, name: str, url: str, params, fee):
|
||||
self.market = market
|
||||
self.name = name
|
||||
self.url = url
|
||||
self.params = params
|
||||
self.fee = fee
|
||||
self.rate = None
|
||||
self._updater = LoopingCall(self._update_price)
|
||||
self._task: asyncio.Task = None
|
||||
self._online = True
|
||||
|
||||
def rate_is_initialized(self):
|
||||
@@ -54,17 +52,16 @@ class MarketFeed:
|
||||
def is_online(self):
|
||||
return self._online
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _make_request(self):
|
||||
response = yield treq.get(self.url, params=self.params, timeout=self.REQUESTS_TIMEOUT)
|
||||
defer.returnValue((yield response.content()))
|
||||
async def _make_request(self):
|
||||
async with aiohttp.request('get', self.url, params=self.params) as response:
|
||||
return (await response.read()).decode()
|
||||
|
||||
def _handle_response(self, response):
|
||||
return NotImplementedError
|
||||
raise NotImplementedError()
|
||||
|
||||
def _subtract_fee(self, from_amount):
|
||||
# increase amount to account for market fees
|
||||
return defer.succeed(from_amount / (1.0 - self.fee))
|
||||
return from_amount / (1.0 - self.fee)
|
||||
|
||||
def _save_price(self, price):
|
||||
log.debug("Saving price update %f for %s from %s" % (price, self.market, self.name))
|
||||
@@ -77,21 +74,23 @@ class MarketFeed:
|
||||
log.debug("Exchange rate error (%s from %s): %s", self.market, self.name, err)
|
||||
self._online = False
|
||||
|
||||
def _update_price(self):
|
||||
d = self._make_request()
|
||||
d.addCallback(self._handle_response)
|
||||
d.addCallback(self._subtract_fee)
|
||||
d.addCallback(self._save_price)
|
||||
d.addErrback(self._on_error)
|
||||
return d
|
||||
async def _update_price(self):
|
||||
while True:
|
||||
try:
|
||||
response = await asyncio.wait_for(self._make_request(), self.REQUESTS_TIMEOUT)
|
||||
self._save_price(self._subtract_fee(self._handle_response(response)))
|
||||
except (asyncio.TimeoutError, InvalidExchangeRateResponse) as err:
|
||||
self._on_error(err)
|
||||
await asyncio.sleep(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
|
||||
|
||||
def start(self):
|
||||
if not self._updater.running:
|
||||
self._updater.start(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
|
||||
if not self._task:
|
||||
self._task = asyncio.create_task(self._update_price())
|
||||
|
||||
def stop(self):
|
||||
if self._updater.running:
|
||||
self._updater.stop()
|
||||
if self._task and not self._task.done():
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
|
||||
|
||||
class BittrexFeed(MarketFeed):
|
||||
@@ -116,7 +115,7 @@ class BittrexFeed(MarketFeed):
|
||||
if totals <= 0 or qtys <= 0:
|
||||
raise InvalidExchangeRateResponse(self.market, 'quantities were not positive')
|
||||
vwap = totals / qtys
|
||||
return defer.succeed(float(1.0 / vwap))
|
||||
return float(1.0 / vwap)
|
||||
|
||||
|
||||
class LBRYioFeed(MarketFeed):
|
||||
@@ -133,7 +132,7 @@ class LBRYioFeed(MarketFeed):
|
||||
json_response = json.loads(response)
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(1.0 / json_response['data']['lbc_btc'])
|
||||
return 1.0 / json_response['data']['lbc_btc']
|
||||
|
||||
|
||||
class LBRYioBTCFeed(MarketFeed):
|
||||
@@ -153,7 +152,7 @@ class LBRYioBTCFeed(MarketFeed):
|
||||
raise InvalidExchangeRateResponse(self.name, "invalid rate response : %s" % response)
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(1.0 / json_response['data']['btc_usd'])
|
||||
return 1.0 / json_response['data']['btc_usd']
|
||||
|
||||
|
||||
class CryptonatorBTCFeed(MarketFeed):
|
||||
@@ -174,7 +173,7 @@ class CryptonatorBTCFeed(MarketFeed):
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(float(json_response['ticker']['price']))
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class CryptonatorFeed(MarketFeed):
|
||||
@@ -195,7 +194,7 @@ class CryptonatorFeed(MarketFeed):
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(float(json_response['ticker']['price']))
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class ExchangeRateManager:
|
||||
@@ -75,7 +75,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['channel_name'] = txo.channel.claim_name
|
||||
try:
|
||||
output['valid_signature'] = claim.validate_signature(
|
||||
txo.get_address(self.ledger), txo.channel.claim
|
||||
txo.get_address(self.ledger), txo.channel.claim, name=txo.claim_name
|
||||
)
|
||||
except BadSignatureError:
|
||||
output['valid_signature'] = False
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
from lbrynet import utils, __version__
|
||||
|
||||
|
||||
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
class HTTPSLogglyHandler(logging.Handler):
|
||||
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
|
||||
super().__init__()
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.cookies = cookies or {}
|
||||
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
|
||||
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
|
||||
)
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
async def _emit(self, record):
|
||||
payload = self.format(record)
|
||||
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
|
||||
def emit(self, record):
|
||||
asyncio.ensure_future(self._emit(record))
|
||||
|
||||
|
||||
def get_loggly_handler(loggly_token=LOGGLY_TOKEN):
|
||||
handler = HTTPSLogglyHandler(loggly_token)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
return handler
|
||||
@@ -3,7 +3,7 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def migrate_db(db_dir, start, end):
|
||||
def migrate_db(conf, start, end):
|
||||
current = start
|
||||
while current < end:
|
||||
if current == 1:
|
||||
@@ -25,7 +25,7 @@ def migrate_db(db_dir, start, end):
|
||||
else:
|
||||
raise Exception("DB migration of version {} to {} is not available".format(current,
|
||||
current+1))
|
||||
do_migration(db_dir)
|
||||
do_migration(conf)
|
||||
current += 1
|
||||
log.info("successfully migrated the database from revision %i to %i", current - 1, current)
|
||||
return None
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
UNSET_NOUT = -1
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blockchainname_db(db_dir)
|
||||
migrate_blockchainname_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blockchainname_db(db_dir)
|
||||
migrate_blockchainname_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blobs_db(db_dir)
|
||||
migrate_blobs_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
add_lbry_file_metadata(db_dir)
|
||||
add_lbry_file_metadata(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import sqlite3
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from lbrynet import conf
|
||||
from binascii import hexlify
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -103,13 +103,13 @@ def verify_sd_blob(sd_hash, blob_dir):
|
||||
return decoded, sd_length
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
new_db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
def do_migration(conf):
|
||||
new_db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(new_db_path)
|
||||
|
||||
metadata_db = sqlite3.connect(os.path.join(db_dir, "blockchainname.db"))
|
||||
lbryfile_db = sqlite3.connect(os.path.join(db_dir, 'lbryfile_info.db'))
|
||||
blobs_db = sqlite3.connect(os.path.join(db_dir, 'blobs.db'))
|
||||
metadata_db = sqlite3.connect(os.path.join(conf.data_dir, "blockchainname.db"))
|
||||
lbryfile_db = sqlite3.connect(os.path.join(conf.data_dir, 'lbryfile_info.db'))
|
||||
blobs_db = sqlite3.connect(os.path.join(conf.data_dir, 'blobs.db'))
|
||||
|
||||
name_metadata_cursor = metadata_db.cursor()
|
||||
lbryfile_cursor = lbryfile_db.cursor()
|
||||
@@ -185,10 +185,14 @@ def do_migration(db_dir):
|
||||
(stream_hash, blob_hash, position, iv)
|
||||
)
|
||||
|
||||
download_dir = conf.download_dir
|
||||
if not isinstance(download_dir, bytes):
|
||||
download_dir = download_dir.encode()
|
||||
|
||||
# insert the file
|
||||
transaction.execute(
|
||||
"insert or ignore into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, stream_name, conf.settings.download_dir.encode('hex'),
|
||||
(stream_hash, stream_name, hexlify(download_dir),
|
||||
data_rate, status)
|
||||
)
|
||||
|
||||
@@ -273,7 +277,7 @@ def do_migration(db_dir):
|
||||
|
||||
# recover damaged streams
|
||||
if damaged_stream_sds:
|
||||
blob_dir = os.path.join(db_dir, "blobfiles")
|
||||
blob_dir = os.path.join(conf.data_dir, "blobfiles")
|
||||
damaged_sds_on_disk = [] if not os.path.isdir(blob_dir) else list({p for p in os.listdir(blob_dir)
|
||||
if p in damaged_stream_sds})
|
||||
for damaged_sd in damaged_sds_on_disk:
|
||||
@@ -311,7 +315,7 @@ def do_migration(db_dir):
|
||||
log.warning("detected a failed previous migration to revision 6, repairing it")
|
||||
connection.close()
|
||||
os.remove(new_db_path)
|
||||
return do_migration(db_dir)
|
||||
return do_migration(conf)
|
||||
raise err
|
||||
|
||||
connection.close()
|
||||
|
||||
@@ -2,8 +2,8 @@ import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
cursor.executescript("alter table blob add last_announced_time integer;")
|
||||
|
||||
@@ -2,8 +2,8 @@ import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
|
||||
@@ -9,9 +9,9 @@ from lbrynet.blob.CryptBlob import CryptBlobInfo
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
blob_dir = os.path.join(db_dir, "blobfiles")
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
blob_dir = os.path.join(conf.data_dir, "blobfiles")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
|
||||
@@ -144,6 +144,7 @@ types_map = {
|
||||
'.pict': 'image/pict',
|
||||
'.rtf': 'application/rtf',
|
||||
'.xul': 'text/xul',
|
||||
'.m4v': 'video/m4v',
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
@@ -151,9 +152,7 @@ types_map = {
|
||||
}
|
||||
|
||||
|
||||
fallback_mime_type = 'application/octet-stream'
|
||||
|
||||
|
||||
def guess_mime_type(path):
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
return types_map.get(ext, fallback_mime_type)
|
||||
default = f'application/x-ext-{ext}' if ext else 'application/octet-stream'
|
||||
return types_map.get(ext, default)
|
||||
+149
-218
@@ -1,17 +1,16 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import traceback
|
||||
import typing
|
||||
from binascii import hexlify, unhexlify
|
||||
from twisted.internet import defer, task, threads
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbrynet import conf
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.blob.CryptBlob import CryptBlobInfo
|
||||
from lbrynet.dht.constants import dataExpireTimeout
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -44,63 +43,23 @@ def _open_file_for_writing(download_directory, suggested_file_name):
|
||||
return os.path.basename(file_path)
|
||||
|
||||
|
||||
def open_file_for_writing(download_directory, suggested_file_name):
|
||||
"""
|
||||
Used to touch the path of a file to be downloaded
|
||||
|
||||
:param download_directory: (str)
|
||||
:param suggested_file_name: (str)
|
||||
:return: (str) basename
|
||||
"""
|
||||
return threads.deferToThread(_open_file_for_writing, download_directory, suggested_file_name)
|
||||
async def open_file_for_writing(download_directory: str, suggested_file_name: str) -> str:
|
||||
""" Used to touch the path of a file to be downloaded. """
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
None, _open_file_for_writing, download_directory, suggested_file_name
|
||||
)
|
||||
|
||||
|
||||
def rerun_if_locked(f):
|
||||
max_attempts = 5
|
||||
|
||||
def rerun(err, rerun_count, *args, **kwargs):
|
||||
connection = args[0]
|
||||
reactor = connection.reactor
|
||||
log.debug("Failed to execute (%s): %s", err, args)
|
||||
if err.check(sqlite3.OperationalError) and "database is locked" in str(err.value):
|
||||
log.warning("database was locked. rerunning %s with args %s, kwargs %s",
|
||||
str(f), str(args), str(kwargs))
|
||||
if rerun_count < max_attempts:
|
||||
delay = 2**rerun_count
|
||||
return task.deferLater(reactor, delay, inner_wrapper, rerun_count + 1, *args, **kwargs)
|
||||
raise err
|
||||
|
||||
def check_needed_rerun(result, rerun_count):
|
||||
if rerun_count:
|
||||
log.info("successfully reran database query")
|
||||
return result
|
||||
|
||||
def inner_wrapper(rerun_count, *args, **kwargs):
|
||||
d = f(*args, **kwargs)
|
||||
d.addCallback(check_needed_rerun, rerun_count)
|
||||
d.addErrback(rerun, rerun_count, *args, **kwargs)
|
||||
return d
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
return inner_wrapper(0, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
async def looping_call(interval, fun):
|
||||
while True:
|
||||
try:
|
||||
await fun()
|
||||
except Exception as e:
|
||||
log.exception('Looping call experienced exception:', exc_info=e)
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
|
||||
class SqliteConnection(adbapi.ConnectionPool):
|
||||
def __init__(self, db_path):
|
||||
super().__init__('sqlite3', db_path, check_same_thread=False)
|
||||
|
||||
@rerun_if_locked
|
||||
def runInteraction(self, interaction, *args, **kw):
|
||||
return super().runInteraction(interaction, *args, **kw)
|
||||
|
||||
@classmethod
|
||||
def set_reactor(cls, reactor):
|
||||
cls.reactor = reactor
|
||||
|
||||
|
||||
class SQLiteStorage:
|
||||
class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
CREATE_TABLES_QUERY = """
|
||||
pragma foreign_keys=on;
|
||||
@@ -173,70 +132,47 @@ class SQLiteStorage:
|
||||
);
|
||||
"""
|
||||
|
||||
def __init__(self, db_dir, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.db_dir = db_dir
|
||||
self._db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
log.info("connecting to database: %s", self._db_path)
|
||||
self.db = SqliteConnection(self._db_path)
|
||||
self.db.set_reactor(reactor)
|
||||
self.clock = reactor
|
||||
|
||||
# used to refresh the claim attributes on a ManagedEncryptedFileDownloader when a
|
||||
# change to the associated content claim occurs. these are added by the file manager
|
||||
# when it loads each file
|
||||
self.content_claim_callbacks = {} # {<stream_hash>: <callable returning a deferred>}
|
||||
def __init__(self, conf: Config, path, loop=None):
|
||||
super().__init__(path)
|
||||
self.conf = conf
|
||||
self.content_claim_callbacks = {}
|
||||
self.check_should_announce_lc = None
|
||||
if 'reflector' not in conf.settings['components_to_skip']:
|
||||
self.check_should_announce_lc = task.LoopingCall(self.verify_will_announce_all_head_and_sd_blobs)
|
||||
self.loop = loop or asyncio.get_event_loop()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
def _create_tables(transaction):
|
||||
transaction.executescript(self.CREATE_TABLES_QUERY)
|
||||
yield self.db.runInteraction(_create_tables)
|
||||
if self.check_should_announce_lc and not self.check_should_announce_lc.running:
|
||||
self.check_should_announce_lc.start(600)
|
||||
defer.returnValue(None)
|
||||
async def open(self):
|
||||
await super().open()
|
||||
if 'reflector' not in self.conf.components_to_skip:
|
||||
self.check_should_announce_lc = looping_call(
|
||||
600, self.verify_will_announce_all_head_and_sd_blobs
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run_and_return_one_or_none(self, query, *args):
|
||||
result = yield self.db.runQuery(query, args)
|
||||
if result:
|
||||
defer.returnValue(result[0][0])
|
||||
else:
|
||||
defer.returnValue(None)
|
||||
async def close(self):
|
||||
if self.check_should_announce_lc is not None:
|
||||
self.check_should_announce_lc.close()
|
||||
await super().close()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run_and_return_list(self, query, *args):
|
||||
result = yield self.db.runQuery(query, args)
|
||||
if result:
|
||||
defer.returnValue([i[0] for i in result])
|
||||
else:
|
||||
defer.returnValue([])
|
||||
async def run_and_return_one_or_none(self, query, *args):
|
||||
for row in await self.db.execute_fetchall(query, args):
|
||||
if len(row) == 1:
|
||||
return row[0]
|
||||
return row
|
||||
|
||||
def run_and_return_id(self, query, *args):
|
||||
def do_save(t):
|
||||
t.execute(query, args)
|
||||
return t.lastrowid
|
||||
return self.db.runInteraction(do_save)
|
||||
async def run_and_return_list(self, query, *args):
|
||||
rows = list(await self.db.execute_fetchall(query, args))
|
||||
return [col[0] for col in rows] if rows else []
|
||||
|
||||
def stop(self):
|
||||
if self.check_should_announce_lc and self.check_should_announce_lc.running:
|
||||
self.check_should_announce_lc.stop()
|
||||
self.db.close()
|
||||
return defer.succeed(True)
|
||||
async def run_and_return_id(self, query, *args):
|
||||
return (await self.db.execute(query, args)).lastrowid
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
def add_completed_blob(self, blob_hash, length, next_announce_time, should_announce, status="finished"):
|
||||
log.debug("Adding a completed blob. blob_hash=%s, length=%i", blob_hash, length)
|
||||
values = (blob_hash, length, next_announce_time or 0, int(bool(should_announce)), status, 0, 0)
|
||||
return self.db.runOperation("insert or replace into blob values (?, ?, ?, ?, ?, ?, ?)", values)
|
||||
return self.db.execute("insert or replace into blob values (?, ?, ?, ?, ?, ?, ?)", values)
|
||||
|
||||
def set_should_announce(self, blob_hash, next_announce_time, should_announce):
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"update blob set next_announce_time=?, should_announce=? where blob_hash=?",
|
||||
(next_announce_time or 0, int(bool(should_announce)), blob_hash)
|
||||
)
|
||||
@@ -246,9 +182,8 @@ class SQLiteStorage:
|
||||
"select status from blob where blob_hash=?", blob_hash
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_known_blob(self, blob_hash, length):
|
||||
yield self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)", (blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
)
|
||||
|
||||
@@ -267,12 +202,11 @@ class SQLiteStorage:
|
||||
"select blob_hash from blob where should_announce=1 and status='finished'"
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_all_finished_blobs(self):
|
||||
blob_hashes = yield self.run_and_return_list(
|
||||
async def get_all_finished_blobs(self):
|
||||
blob_hashes = await self.run_and_return_list(
|
||||
"select blob_hash from blob where status='finished'"
|
||||
)
|
||||
defer.returnValue([unhexlify(blob_hash) for blob_hash in blob_hashes])
|
||||
return [unhexlify(blob_hash) for blob_hash in blob_hashes]
|
||||
|
||||
def count_finished_blobs(self):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -280,14 +214,14 @@ class SQLiteStorage:
|
||||
)
|
||||
|
||||
def update_last_announced_blob(self, blob_hash, last_announced):
|
||||
return self.db.runOperation(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 where blob_hash=?",
|
||||
(int(last_announced + (dataExpireTimeout / 2)), int(last_announced), blob_hash)
|
||||
)
|
||||
return self.db.execute(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 where blob_hash=?",
|
||||
(int(last_announced + (dataExpireTimeout / 2)), int(last_announced), blob_hash)
|
||||
)
|
||||
|
||||
def should_single_announce_blobs(self, blob_hashes, immediate=False):
|
||||
def set_single_announce(transaction):
|
||||
now = self.clock.seconds()
|
||||
now = self.loop.time()
|
||||
for blob_hash in blob_hashes:
|
||||
if immediate:
|
||||
transaction.execute(
|
||||
@@ -298,12 +232,12 @@ class SQLiteStorage:
|
||||
transaction.execute(
|
||||
"update blob set single_announce=1 where blob_hash=? and status='finished'", (blob_hash, )
|
||||
)
|
||||
return self.db.runInteraction(set_single_announce)
|
||||
return self.db.run(set_single_announce)
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
def get_and_update(transaction):
|
||||
timestamp = self.clock.seconds()
|
||||
if conf.settings['announce_head_blobs_only']:
|
||||
timestamp = self.loop.time()
|
||||
if self.conf.announce_head_blobs_only:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob "
|
||||
"where blob_hash is not null and "
|
||||
@@ -317,13 +251,13 @@ class SQLiteStorage:
|
||||
)
|
||||
blobs = [b[0] for b in r.fetchall()]
|
||||
return blobs
|
||||
return self.db.runInteraction(get_and_update)
|
||||
return self.db.run(get_and_update)
|
||||
|
||||
def delete_blobs_from_db(self, blob_hashes):
|
||||
def delete_blobs(transaction):
|
||||
for blob_hash in blob_hashes:
|
||||
transaction.execute("delete from blob where blob_hash=?;", (blob_hash,))
|
||||
return self.db.runInteraction(delete_blobs)
|
||||
return self.db.run(delete_blobs)
|
||||
|
||||
def get_all_blob_hashes(self):
|
||||
return self.run_and_return_list("select blob_hash from blob")
|
||||
@@ -336,17 +270,16 @@ class SQLiteStorage:
|
||||
transaction.execute("insert into stream_blob values (?, ?, ?, ?)",
|
||||
(stream_hash, blob_info.get('blob_hash', None),
|
||||
blob_info['blob_num'], blob_info['iv']))
|
||||
return self.db.runInteraction(_add_stream_blobs)
|
||||
return self.db.run(_add_stream_blobs)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_known_blobs(self, blob_infos):
|
||||
async def add_known_blobs(self, blob_infos):
|
||||
for blob_info in blob_infos:
|
||||
if blob_info.get('blob_hash') and blob_info['length']:
|
||||
yield self.add_known_blob(blob_info['blob_hash'], blob_info['length'])
|
||||
await self.add_known_blob(blob_info['blob_hash'], blob_info['length'])
|
||||
|
||||
def verify_will_announce_head_and_sd_blobs(self, stream_hash):
|
||||
# fix should_announce for imported head and sd blobs
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"update blob set should_announce=1 "
|
||||
"where should_announce=0 and "
|
||||
"blob.blob_hash in "
|
||||
@@ -358,7 +291,7 @@ class SQLiteStorage:
|
||||
)
|
||||
|
||||
def verify_will_announce_all_head_and_sd_blobs(self):
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"update blob set should_announce=1 "
|
||||
"where should_announce=0 and "
|
||||
"blob.blob_hash in "
|
||||
@@ -383,23 +316,24 @@ class SQLiteStorage:
|
||||
:param stream_blob_infos: (list) of blob info dictionaries
|
||||
:return: (defer.Deferred)
|
||||
"""
|
||||
|
||||
def _store_stream(transaction):
|
||||
transaction.execute("insert into stream values (?, ?, ?, ?, ?);",
|
||||
(stream_hash, sd_hash, stream_key, stream_name,
|
||||
suggested_file_name))
|
||||
|
||||
transaction.execute(
|
||||
"insert into stream values (?, ?, ?, ?, ?);", (
|
||||
stream_hash, sd_hash, stream_key, stream_name, suggested_file_name
|
||||
)
|
||||
)
|
||||
for blob_info in stream_blob_infos:
|
||||
transaction.execute("insert into stream_blob values (?, ?, ?, ?)",
|
||||
(stream_hash, blob_info.get('blob_hash', None),
|
||||
blob_info['blob_num'], blob_info['iv']))
|
||||
transaction.execute(
|
||||
"insert into stream_blob values (?, ?, ?, ?)", (
|
||||
stream_hash, blob_info.get('blob_hash', None),
|
||||
blob_info['blob_num'], blob_info['iv']
|
||||
)
|
||||
)
|
||||
return self.db.run(_store_stream)
|
||||
|
||||
return self.db.runInteraction(_store_stream)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_stream(self, stream_hash):
|
||||
sd_hash = yield self.get_sd_blob_hash_for_stream(stream_hash)
|
||||
stream_blobs = yield self.get_blobs_for_stream(stream_hash)
|
||||
async def delete_stream(self, stream_hash):
|
||||
sd_hash = await self.get_sd_blob_hash_for_stream(stream_hash)
|
||||
stream_blobs = await self.get_blobs_for_stream(stream_hash)
|
||||
blob_hashes = [b.blob_hash for b in stream_blobs if b.blob_hash is not None]
|
||||
|
||||
def _delete_stream(transaction):
|
||||
@@ -410,21 +344,25 @@ class SQLiteStorage:
|
||||
transaction.execute("delete from blob where blob_hash=?", (sd_hash, ))
|
||||
for blob_hash in blob_hashes:
|
||||
transaction.execute("delete from blob where blob_hash=?;", (blob_hash, ))
|
||||
yield self.db.runInteraction(_delete_stream)
|
||||
|
||||
await self.db.run(_delete_stream)
|
||||
|
||||
def get_all_streams(self):
|
||||
return self.run_and_return_list("select stream_hash from stream")
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
d = self.db.runQuery("select stream_name, stream_key, suggested_filename, sd_hash from stream "
|
||||
"where stream_hash=?", (stream_hash, ))
|
||||
d.addCallback(lambda r: None if not r else r[0])
|
||||
return d
|
||||
return self.run_and_return_one_or_none(
|
||||
"select stream_name, stream_key, suggested_filename, sd_hash from stream "
|
||||
"where stream_hash=?", stream_hash
|
||||
)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
d = self.db.runQuery("select stream_hash from stream where stream_hash=?", (stream_hash, ))
|
||||
d.addCallback(lambda r: bool(len(r)))
|
||||
return d
|
||||
async def check_if_stream_exists(self, stream_hash):
|
||||
row = await self.run_and_return_one_or_none(
|
||||
"select stream_hash from stream where stream_hash=?", stream_hash
|
||||
)
|
||||
if row is not None:
|
||||
return bool(len(row))
|
||||
return False
|
||||
|
||||
def get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -466,7 +404,7 @@ class SQLiteStorage:
|
||||
crypt_blob_infos.append(CryptBlobInfo(blob_hash, position, blob_length, iv))
|
||||
crypt_blob_infos = sorted(crypt_blob_infos, key=lambda info: info.blob_num)
|
||||
return crypt_blob_infos
|
||||
return self.db.runInteraction(_get_blobs_for_stream)
|
||||
return self.db.run(_get_blobs_for_stream)
|
||||
|
||||
def get_pending_blobs_for_stream(self, stream_hash):
|
||||
return self.run_and_return_list(
|
||||
@@ -493,14 +431,12 @@ class SQLiteStorage:
|
||||
|
||||
# # # # # # # # # file stuff # # # # # # # # #
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
|
||||
async def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
|
||||
# touch the closest available file to the file name
|
||||
file_name = yield open_file_for_writing(unhexlify(download_directory).decode(), unhexlify(file_name).decode())
|
||||
result = yield self.save_published_file(
|
||||
file_name = await open_file_for_writing(unhexlify(download_directory).decode(), unhexlify(file_name).decode())
|
||||
return await self.save_published_file(
|
||||
stream_hash, hexlify(file_name.encode()), download_directory, data_payment_rate
|
||||
)
|
||||
defer.returnValue(result)
|
||||
|
||||
def save_published_file(self, stream_hash, file_name, download_directory, data_payment_rate, status="stopped"):
|
||||
return self.run_and_return_id(
|
||||
@@ -509,7 +445,9 @@ class SQLiteStorage:
|
||||
)
|
||||
|
||||
def get_filename_for_rowid(self, rowid):
|
||||
return self.run_and_return_one_or_none("select file_name from file where rowid=?", rowid)
|
||||
return self.run_and_return_one_or_none(
|
||||
"select file_name from file where rowid=?", rowid
|
||||
)
|
||||
|
||||
def get_all_lbry_files(self):
|
||||
def _lbry_file_dict(rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
@@ -535,13 +473,11 @@ class SQLiteStorage:
|
||||
).fetchall()
|
||||
]
|
||||
|
||||
d = self.db.runInteraction(_get_all_files)
|
||||
return d
|
||||
return self.db.run(_get_all_files)
|
||||
|
||||
def change_file_status(self, rowid, new_status):
|
||||
d = self.db.runQuery("update file set status=? where rowid=?", (new_status, rowid))
|
||||
d.addCallback(lambda _: new_status)
|
||||
return d
|
||||
async def change_file_status(self, rowid, new_status):
|
||||
await self.db.execute("update file set status=? where rowid=?", (new_status, rowid))
|
||||
return new_status
|
||||
|
||||
def get_lbry_file_status(self, rowid):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -565,7 +501,7 @@ class SQLiteStorage:
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
return self.db.runInteraction(_save_support)
|
||||
return self.db.run(_save_support)
|
||||
|
||||
def get_supports(self, *claim_ids):
|
||||
def _format_support(outpoint, supported_id, amount, address):
|
||||
@@ -587,15 +523,16 @@ class SQLiteStorage:
|
||||
)
|
||||
]
|
||||
|
||||
return self.db.runInteraction(_get_supports)
|
||||
return self.db.run(_get_supports)
|
||||
|
||||
# # # # # # # # # claim functions # # # # # # # # #
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def save_claims(self, claim_infos):
|
||||
async def save_claims(self, claim_infos):
|
||||
support_callbacks = []
|
||||
update_file_callbacks = []
|
||||
|
||||
def _save_claims(transaction):
|
||||
content_claims_to_update = []
|
||||
support_callbacks = []
|
||||
for claim_info in claim_infos:
|
||||
outpoint = "%s:%i" % (claim_info['txid'], claim_info['nout'])
|
||||
claim_id = claim_info['claim_id']
|
||||
@@ -605,7 +542,7 @@ class SQLiteStorage:
|
||||
address = claim_info['address']
|
||||
sequence = claim_info['claim_sequence']
|
||||
try:
|
||||
certificate_id = claim_info['value'].get('content_claims_to_update', {}).get('certificateId')
|
||||
certificate_id = claim_info['value'].get('publisherSignature', {}).get('certificateId')
|
||||
except AttributeError:
|
||||
certificate_id = None
|
||||
try:
|
||||
@@ -622,7 +559,7 @@ class SQLiteStorage:
|
||||
)
|
||||
if 'supports' in claim_info: # if this response doesn't have support info don't overwrite the existing
|
||||
# support info
|
||||
support_callbacks.append(self.save_supports(claim_id, claim_info['supports']))
|
||||
support_callbacks.append((claim_id, claim_info['supports']))
|
||||
if not source_hash:
|
||||
continue
|
||||
stream_hash = transaction.execute(
|
||||
@@ -644,18 +581,18 @@ class SQLiteStorage:
|
||||
content_claims_to_update.append((stream_hash, outpoint))
|
||||
elif known_outpoint != outpoint:
|
||||
content_claims_to_update.append((stream_hash, outpoint))
|
||||
update_file_callbacks = []
|
||||
for stream_hash, outpoint in content_claims_to_update:
|
||||
self._save_content_claim(transaction, outpoint, stream_hash)
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
update_file_callbacks.append(self.content_claim_callbacks[stream_hash]())
|
||||
return update_file_callbacks, support_callbacks
|
||||
|
||||
content_dl, support_dl = yield self.db.runInteraction(_save_claims)
|
||||
if content_dl:
|
||||
yield defer.DeferredList(content_dl)
|
||||
if support_dl:
|
||||
yield defer.DeferredList(support_dl)
|
||||
await self.db.run(_save_claims)
|
||||
if update_file_callbacks:
|
||||
await asyncio.wait(update_file_callbacks)
|
||||
if support_callbacks:
|
||||
await asyncio.wait([
|
||||
self.save_supports(*args) for args in support_callbacks
|
||||
])
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = []
|
||||
@@ -718,16 +655,13 @@ class SQLiteStorage:
|
||||
# update the claim associated to the file
|
||||
transaction.execute("insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def save_content_claim(self, stream_hash, claim_outpoint):
|
||||
yield self.db.runInteraction(self._save_content_claim, claim_outpoint, stream_hash)
|
||||
async def save_content_claim(self, stream_hash, claim_outpoint):
|
||||
await self.db.run(self._save_content_claim, claim_outpoint, stream_hash)
|
||||
# update corresponding ManagedEncryptedFileDownloader object
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
file_callback = self.content_claim_callbacks[stream_hash]
|
||||
yield file_callback()
|
||||
await self.content_claim_callbacks[stream_hash]()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_content_claim(self, stream_hash, include_supports=True):
|
||||
async def get_content_claim(self, stream_hash, include_supports=True):
|
||||
def _get_claim_from_stream_hash(transaction):
|
||||
claim_info = transaction.execute(
|
||||
"select c.*, "
|
||||
@@ -745,15 +679,13 @@ class SQLiteStorage:
|
||||
result['channel_name'] = channel_name
|
||||
return result
|
||||
|
||||
result = yield self.db.runInteraction(_get_claim_from_stream_hash)
|
||||
result = await self.db.run(_get_claim_from_stream_hash)
|
||||
if result and include_supports:
|
||||
supports = yield self.get_supports(result['claim_id'])
|
||||
result['supports'] = supports
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], supports)
|
||||
defer.returnValue(result)
|
||||
result['supports'] = await self.get_supports(result['claim_id'])
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], result['supports'])
|
||||
return result
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claims_from_stream_hashes(self, stream_hashes, include_supports=True):
|
||||
async def get_claims_from_stream_hashes(self, stream_hashes, include_supports=True):
|
||||
def _batch_get_claim(transaction):
|
||||
results = {}
|
||||
claim_infos = _batched_select(
|
||||
@@ -781,10 +713,10 @@ class SQLiteStorage:
|
||||
results[stream_hash]['channel_name'] = channel_name
|
||||
return results
|
||||
|
||||
claims = yield self.db.runInteraction(_batch_get_claim)
|
||||
claims = await self.db.run(_batch_get_claim)
|
||||
if include_supports:
|
||||
all_supports = {}
|
||||
for support in (yield self.get_supports(*[claim['claim_id'] for claim in claims.values()])):
|
||||
for support in await self.get_supports(*[claim['claim_id'] for claim in claims.values()]):
|
||||
all_supports.setdefault(support['claim_id'], []).append(support)
|
||||
for stream_hash in claims.keys():
|
||||
claim = claims[stream_hash]
|
||||
@@ -792,28 +724,28 @@ class SQLiteStorage:
|
||||
claim['supports'] = supports
|
||||
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
|
||||
claims[stream_hash] = claim
|
||||
defer.returnValue(claims)
|
||||
return claims
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claim(self, claim_outpoint, include_supports=True):
|
||||
async def get_claim(self, claim_outpoint, include_supports=True):
|
||||
def _get_claim(transaction):
|
||||
claim_info = transaction.execute("select c.*, "
|
||||
"case when c.channel_claim_id is not null then "
|
||||
"(select claim_name from claim where claim_id==c.channel_claim_id) "
|
||||
"else null end as channel_name from claim c where claim_outpoint = ?",
|
||||
(claim_outpoint,)).fetchone()
|
||||
claim_info = transaction.execute(
|
||||
"select c.*, "
|
||||
"case when c.channel_claim_id is not null then "
|
||||
"(select claim_name from claim where claim_id==c.channel_claim_id) "
|
||||
"else null end as channel_name from claim c where claim_outpoint = ?",
|
||||
(claim_outpoint,)
|
||||
).fetchone()
|
||||
channel_name = claim_info[-1]
|
||||
result = _format_claim_response(*claim_info[:-1])
|
||||
if channel_name:
|
||||
result['channel_name'] = channel_name
|
||||
return result
|
||||
|
||||
result = yield self.db.runInteraction(_get_claim)
|
||||
result = await self.db.run(_get_claim)
|
||||
if include_supports:
|
||||
supports = yield self.get_supports(result['claim_id'])
|
||||
result['supports'] = supports
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], supports)
|
||||
defer.returnValue(result)
|
||||
result['supports'] = await self.get_supports(result['claim_id'])
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], result['supports'])
|
||||
return result
|
||||
|
||||
def get_unknown_certificate_ids(self):
|
||||
def _get_unknown_certificate_claim_ids(transaction):
|
||||
@@ -825,11 +757,10 @@ class SQLiteStorage:
|
||||
"(select c2.claim_id from claim as c2)"
|
||||
).fetchall()
|
||||
]
|
||||
return self.db.runInteraction(_get_unknown_certificate_claim_ids)
|
||||
return self.db.run(_get_unknown_certificate_claim_ids)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_pending_claim_outpoints(self):
|
||||
claim_outpoints = yield self.run_and_return_list("select claim_outpoint from claim where height=-1")
|
||||
async def get_pending_claim_outpoints(self):
|
||||
claim_outpoints = await self.run_and_return_list("select claim_outpoint from claim where height=-1")
|
||||
results = {} # {txid: [nout, ...]}
|
||||
for outpoint_str in claim_outpoints:
|
||||
txid, nout = outpoint_str.split(":")
|
||||
@@ -838,7 +769,7 @@ class SQLiteStorage:
|
||||
results[txid] = outputs
|
||||
if results:
|
||||
log.debug("missing transaction heights for %i claims", len(results))
|
||||
defer.returnValue(results)
|
||||
return results
|
||||
|
||||
def save_claim_tx_heights(self, claim_tx_heights):
|
||||
def _save_claim_heights(transaction):
|
||||
@@ -847,17 +778,17 @@ class SQLiteStorage:
|
||||
"update claim set height=? where claim_outpoint=? and height=-1",
|
||||
(height, outpoint)
|
||||
)
|
||||
return self.db.runInteraction(_save_claim_heights)
|
||||
return self.db.run(_save_claim_heights)
|
||||
|
||||
# # # # # # # # # reflector functions # # # # # # # # #
|
||||
|
||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||
if success:
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.clock.seconds())
|
||||
(sd_hash, reflector_address, self.loop.time())
|
||||
)
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
(sd_hash, reflector_address)
|
||||
)
|
||||
@@ -867,7 +798,7 @@ class SQLiteStorage:
|
||||
"select s.sd_hash from stream s "
|
||||
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
|
||||
"where r.timestamp is null or r.timestamp < ?",
|
||||
self.clock.seconds() - conf.settings['auto_re_reflect_interval']
|
||||
self.loop.time() - self.conf.auto_re_reflect_interval
|
||||
)
|
||||
|
||||
|
||||
|
||||
File renamed without changes.
@@ -165,15 +165,10 @@ class BlobReflectorClient(Protocol):
|
||||
blob_hash = self.blob_hashes_to_send[0]
|
||||
log.debug('No current blob, sending the next one: %s', blob_hash)
|
||||
self.blob_hashes_to_send = self.blob_hashes_to_send[1:]
|
||||
d = self.blob_manager.get_blob(blob_hash)
|
||||
d.addCallback(self.open_blob_for_reading)
|
||||
blob = self.blob_manager.get_blob(blob_hash)
|
||||
self.open_blob_for_reading(blob)
|
||||
# send the server the next blob hash + length
|
||||
d.addCallbacks(
|
||||
lambda _: self.send_blob_info(),
|
||||
errback=log.fail(self.disconnect),
|
||||
errbackArgs=("Error reflecting blob %s", blob_hash)
|
||||
)
|
||||
return d
|
||||
return self.send_blob_info()
|
||||
else:
|
||||
# close connection
|
||||
log.debug('No more blob hashes, closing connection')
|
||||
|
||||
@@ -6,6 +6,7 @@ from twisted.protocols.basic import FileSender
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.internet import defer, error
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.reflector.common import IncompleteResponse, ReflectorRequestError
|
||||
from lbrynet.extras.reflector.common import REFLECTOR_V1, REFLECTOR_V2
|
||||
|
||||
@@ -61,7 +62,9 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
else:
|
||||
reflected = False
|
||||
|
||||
d = self.blob_manager.storage.update_reflected_stream(self.sd_hash, self.transport.getPeer().host, reflected)
|
||||
d = f2d(self.blob_manager.storage.update_reflected_stream(
|
||||
self.sd_hash, self.transport.getPeer().host, reflected
|
||||
))
|
||||
d.addCallback(lambda _: result)
|
||||
return d
|
||||
|
||||
@@ -113,10 +116,7 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
for crypt_blob in blobs:
|
||||
if crypt_blob.blob_hash and crypt_blob.length:
|
||||
yield self.blob_manager.get_blob(crypt_blob.blob_hash, crypt_blob.length)
|
||||
|
||||
dl = defer.DeferredList(list(get_blobs(blobs_in_stream)), consumeErrors=True)
|
||||
dl.addCallback(lambda blobs: [blob for r, blob in blobs if r and blob.get_is_verified()])
|
||||
return dl
|
||||
return [blob for blob in get_blobs(blobs_in_stream) if blob.get_is_verified()]
|
||||
|
||||
def set_blobs_to_send(self, blobs_to_send):
|
||||
for blob in blobs_to_send:
|
||||
@@ -132,7 +132,7 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
len(filtered))
|
||||
return filtered
|
||||
|
||||
d = self.factory.blob_manager.storage.get_blobs_for_stream(self.stream_hash)
|
||||
d = f2d(self.factory.blob_manager.storage.get_blobs_for_stream(self.stream_hash))
|
||||
d.addCallback(self.get_validated_blobs)
|
||||
if not self.descriptor_needed:
|
||||
d.addCallback(lambda filtered:
|
||||
@@ -303,7 +303,7 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
# open the next blob to send
|
||||
blob_hash = self.blob_hashes_to_send[0]
|
||||
self.blob_hashes_to_send = self.blob_hashes_to_send[1:]
|
||||
d = self.blob_manager.get_blob(blob_hash)
|
||||
d = defer.succeed(self.blob_manager.get_blob(blob_hash))
|
||||
d.addCallback(self.open_blob_for_reading)
|
||||
d.addCallbacks(lambda _: self.send_blob_info(),
|
||||
lambda err: self.skip_missing_blob(err, blob.blob_hash))
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
import random
|
||||
|
||||
from twisted.internet import reactor, defer
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory
|
||||
from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory
|
||||
|
||||
@@ -45,43 +42,22 @@ def _reflect_blobs(blob_manager, blob_hashes, reflector_server):
|
||||
ip = yield resolve(reflector_address)
|
||||
yield reactor.connectTCP(ip, reflector_port, factory)
|
||||
result = yield factory.finished_deferred
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
|
||||
|
||||
def reflect_file(lbry_file, reflector_server=None):
|
||||
if reflector_server:
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = reflector_server, 5566
|
||||
def reflect_file(lbry_file, reflector_server):
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = random.choice(conf.settings['reflector_servers'])
|
||||
reflector_server = reflector_server, 5566
|
||||
return _reflect_file(lbry_file, reflector_server)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def reflect_stream(blob_manager, stream_hash, reflector_server=None):
|
||||
if reflector_server:
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = reflector_server, 5566
|
||||
def reflect_blob_hashes(blob_hashes, blob_manager, reflector_server):
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = random.choice(conf.settings['reflector_servers'])
|
||||
sd_hash = yield blob_manager.storage.get_sd_blob_hash_for_stream(stream_hash)
|
||||
result = yield _reflect_stream(blob_manager, stream_hash, sd_hash, reflector_server)
|
||||
defer.returnValue(result)
|
||||
|
||||
|
||||
def reflect_blob_hashes(blob_hashes, blob_manager, reflector_server=None):
|
||||
if reflector_server:
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = reflector_server, 5566
|
||||
else:
|
||||
reflector_server = random.choice(conf.settings['reflector_servers'])
|
||||
reflector_server = reflector_server, 5566
|
||||
return _reflect_blobs(blob_manager, blob_hashes, reflector_server)
|
||||
@@ -236,3 +236,6 @@ class Account(BaseAccount):
|
||||
)
|
||||
|
||||
return tx
|
||||
|
||||
async def release_all_outputs(self):
|
||||
await self.ledger.db.release_all_outputs(self)
|
||||
@@ -117,3 +117,11 @@ class WalletDatabase(BaseDatabase):
|
||||
channel.private_key = private_key
|
||||
certificates.append(channel)
|
||||
return certificates
|
||||
|
||||
async def release_all_outputs(self, account):
|
||||
await self.db.execute(
|
||||
"UPDATE txo SET is_reserved = 0 WHERE"
|
||||
" is_reserved = 1 AND txo.address IN ("
|
||||
" SELECT address from pubkey_address WHERE account = ?"
|
||||
" )", [account.public_key.address]
|
||||
)
|
||||
@@ -65,8 +65,9 @@ class MainNetLedger(BaseLedger):
|
||||
except Exception as e:
|
||||
return {'error': str(e)}
|
||||
try:
|
||||
resolver = self.resolver
|
||||
resolutions = await self.network.get_values_for_uris(self.headers.hash().decode(), *uris)
|
||||
return await self.resolver._handle_resolutions(resolutions, uris, page, page_size)
|
||||
return await resolver._handle_resolutions(resolutions, uris, page, page_size)
|
||||
except Exception as e:
|
||||
return {'error': str(e)}
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import os
|
||||
import json
|
||||
import asyncio
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
|
||||
@@ -15,6 +14,7 @@ from torba.rpc.jsonrpc import CodeMessageError
|
||||
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.wallet.ledger import MainNetLedger
|
||||
from lbrynet.extras.wallet.account import BaseAccount, generate_certificate
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
@@ -166,14 +166,12 @@ class LbryWalletManager(BaseWalletManager):
|
||||
'lbrycrd_main': 'lbc_mainnet',
|
||||
'lbrycrd_testnet': 'lbc_testnet',
|
||||
'lbrycrd_regtest': 'lbc_regtest'
|
||||
}[settings['blockchain_name']]
|
||||
}[settings.blockchain_name]
|
||||
|
||||
ledger_config = {
|
||||
'auto_connect': True,
|
||||
'default_servers': settings['lbryum_servers'],
|
||||
'default_servers': settings.lbryum_servers,
|
||||
'data_path': settings.wallet_dir,
|
||||
'use_keyring': settings['use_keyring'],
|
||||
#'db': db
|
||||
}
|
||||
|
||||
wallets_directory = os.path.join(settings.wallet_dir, 'wallets')
|
||||
@@ -277,7 +275,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
if 'error' not in results:
|
||||
await self.old_db.save_claims_for_resolve([
|
||||
value for value in results.values() if 'error' not in value
|
||||
]).asFuture(asyncio.get_event_loop())
|
||||
])
|
||||
return results
|
||||
|
||||
async def get_claims_for_name(self, name: str):
|
||||
@@ -413,7 +411,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
claim_address = await account.receiving.get_or_create_usable_address()
|
||||
if certificate:
|
||||
claim = claim.sign(
|
||||
certificate.private_key, claim_address, certificate.claim_id, curve=SECP256k1
|
||||
certificate.private_key, claim_address, certificate.claim_id, curve=SECP256k1, name=name
|
||||
)
|
||||
existing_claims = await account.get_claims(
|
||||
claim_name_type__any={'is_claim': 1, 'is_update': 1}, # exclude is_supports
|
||||
@@ -432,7 +430,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.old_db.save_claims([self._old_get_temp_claim_info(
|
||||
tx, tx.outputs[0], claim_address, claim_dict, name, dewies_to_lbc(amount)
|
||||
)]).asFuture(asyncio.get_event_loop())
|
||||
)])
|
||||
# TODO: release reserved tx outputs in case anything fails by this point
|
||||
return tx
|
||||
|
||||
@@ -446,7 +444,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
'address': holding_address,
|
||||
'claim_id': claim_id,
|
||||
'amount': dewies_to_lbc(amount)
|
||||
}]).asFuture(asyncio.get_event_loop())
|
||||
}])
|
||||
return tx
|
||||
|
||||
async def tip_claim(self, amount, claim_id, account):
|
||||
@@ -461,7 +459,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
'address': claim_to_tip['address'],
|
||||
'claim_id': claim_id,
|
||||
'amount': dewies_to_lbc(amount)
|
||||
}]).asFuture(asyncio.get_event_loop())
|
||||
}])
|
||||
return tx
|
||||
|
||||
async def abandon_claim(self, claim_id, txid, nout, account):
|
||||
@@ -484,7 +482,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
|
||||
await self.old_db.save_claims([self._old_get_temp_claim_info(
|
||||
tx, tx.outputs[0], address, cert, channel_name, dewies_to_lbc(amount)
|
||||
)]).asFuture(asyncio.get_event_loop())
|
||||
)])
|
||||
return tx
|
||||
|
||||
def _old_get_temp_claim_info(self, tx, txo, address, claim_dict, name, bid):
|
||||
@@ -611,8 +609,7 @@ class LBRYcrdAddressQueryHandler:
|
||||
@defer.inlineCallbacks
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
future = self.wallet.get_unused_address_for_peer(self.peer)
|
||||
address = yield defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
address = yield f2d(self.wallet.get_unused_address_for_peer(self.peer))
|
||||
self.address = address
|
||||
fields = {'lbrycrd_address': address}
|
||||
return fields
|
||||
|
||||
@@ -155,8 +155,8 @@ class Resolver:
|
||||
certificate = None
|
||||
certificate_id = smart_decode(claim_result['value']).certificate_id
|
||||
if certificate_id:
|
||||
certificate = await self.network.get_claims_by_ids(certificate_id.decode())
|
||||
certificate = certificate.pop(certificate_id.decode()) if certificate else None
|
||||
certificate = await self.network.get_claims_by_ids(certificate_id)
|
||||
certificate = certificate.pop(certificate_id) if certificate else None
|
||||
return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)
|
||||
|
||||
async def parse_and_validate_claim_result(self, claim_result, certificate=None, raw=False):
|
||||
@@ -180,13 +180,13 @@ class Resolver:
|
||||
if decoded.has_signature:
|
||||
if certificate is None:
|
||||
log.info("fetching certificate to check claim signature")
|
||||
certificate = await self.network.get_claims_by_ids(decoded.certificate_id.decode())
|
||||
certificate = await self.network.get_claims_by_ids(decoded.certificate_id)
|
||||
if not certificate:
|
||||
log.warning('Certificate %s not found', decoded.certificate_id)
|
||||
claim_result['has_signature'] = True
|
||||
claim_result['signature_is_valid'] = False
|
||||
validated, channel_name = validate_claim_signature_and_get_channel_name(
|
||||
decoded, certificate, claim_result['address'])
|
||||
decoded, certificate, claim_result['address'], claim_result['name'])
|
||||
claim_result['channel_name'] = channel_name
|
||||
if validated:
|
||||
claim_result['signature_is_valid'] = True
|
||||
@@ -197,7 +197,7 @@ class Resolver:
|
||||
if 'amount' in claim_result:
|
||||
claim_result = format_amount_value(claim_result)
|
||||
|
||||
claim_result['permanent_url'] = _get_permanent_url(claim_result)
|
||||
claim_result['permanent_url'] = _get_permanent_url(claim_result, decoded.certificate_id)
|
||||
|
||||
return claim_result
|
||||
|
||||
@@ -307,18 +307,11 @@ def format_amount_value(obj):
|
||||
return obj
|
||||
|
||||
|
||||
def _get_permanent_url(claim_result):
|
||||
if claim_result.get('has_signature') and claim_result.get('channel_name'):
|
||||
return "{}#{}/{}".format(
|
||||
claim_result['channel_name'],
|
||||
claim_result['value']['publisherSignature']['certificateId'],
|
||||
claim_result['name']
|
||||
)
|
||||
def _get_permanent_url(claim_result, certificate_id):
|
||||
if certificate_id:
|
||||
return f"{claim_result['channel_name']}#{certificate_id}/{claim_result['name']}"
|
||||
else:
|
||||
return "{}#{}".format(
|
||||
claim_result['name'],
|
||||
claim_result['claim_id']
|
||||
)
|
||||
return f"{claim_result['name']}#{claim_result['claim_id']}"
|
||||
|
||||
|
||||
def _verify_proof(name, claim_trie_root, result, height, depth, transaction_class, hash160_to_address):
|
||||
@@ -386,7 +379,7 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
|
||||
|
||||
|
||||
def validate_claim_signature_and_get_channel_name(claim, certificate_claim,
|
||||
claim_address, decoded_certificate=None):
|
||||
claim_address, name, decoded_certificate=None):
|
||||
if not certificate_claim:
|
||||
return False, None
|
||||
if 'value' not in certificate_claim:
|
||||
@@ -395,18 +388,18 @@ def validate_claim_signature_and_get_channel_name(claim, certificate_claim,
|
||||
certificate = decoded_certificate or smart_decode(certificate_claim['value'])
|
||||
if not isinstance(certificate, ClaimDict):
|
||||
raise TypeError("Certificate is not a ClaimDict: %s" % str(type(certificate)))
|
||||
if _validate_signed_claim(claim, claim_address, certificate):
|
||||
if _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
return True, certificate_claim['name']
|
||||
return False, None
|
||||
|
||||
|
||||
def _validate_signed_claim(claim, claim_address, certificate):
|
||||
def _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
if not claim.has_signature:
|
||||
raise Exception("Claim is not signed")
|
||||
if not is_address(claim_address):
|
||||
raise Exception("Not given a valid claim address")
|
||||
try:
|
||||
if claim.validate_signature(claim_address, certificate.protobuf):
|
||||
if claim.validate_signature(claim_address, certificate.protobuf, name):
|
||||
return True
|
||||
except BadSignatureError:
|
||||
# print_msg("Signature for %s is invalid" % claim_id)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import hashlib
|
||||
import struct
|
||||
from binascii import unhexlify
|
||||
|
||||
import msgpack
|
||||
from torba.server.hash import hash_to_hex_str
|
||||
|
||||
from torba.server.block_processor import BlockProcessor
|
||||
from lbrynet.schema.proto.claim_pb2 import Claim
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
|
||||
@@ -131,9 +131,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
super().backup_blocks(raw_blocks=raw_blocks)
|
||||
self.db.batched_flush_claims()
|
||||
|
||||
def shutdown(self):
|
||||
self.db.shutdown()
|
||||
|
||||
async def flush(self, flush_utxos):
|
||||
self.db.batched_flush_claims()
|
||||
return await super().flush(flush_utxos)
|
||||
@@ -153,14 +150,14 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
def _checksig(self, name, value, address):
|
||||
try:
|
||||
parse_lbry_uri(name.decode()) # skip invalid names
|
||||
cert_id = Claim.FromString(value).publisherSignature.certificateId[::-1] or None
|
||||
claim_dict = smart_decode(value)
|
||||
cert_id = unhexlify(claim_dict.certificate_id)[::-1]
|
||||
if not self.should_validate_signatures:
|
||||
return cert_id
|
||||
if cert_id:
|
||||
cert_claim = self.db.get_claim_info(cert_id)
|
||||
if cert_claim:
|
||||
certificate = smart_decode(cert_claim.value)
|
||||
claim_dict = smart_decode(value)
|
||||
claim_dict.validate_signature(address, certificate)
|
||||
return cert_id
|
||||
except Exception as e:
|
||||
|
||||
@@ -137,3 +137,8 @@ class LBCRegTest(LBC):
|
||||
XPRV_VERBYTES = bytes.fromhex('04358394')
|
||||
P2PKH_VERBYTE = bytes.fromhex("6f")
|
||||
P2SH_VERBYTES = bytes.fromhex("c4")
|
||||
|
||||
|
||||
class LBCTestNet(LBCRegTest):
|
||||
NET = "testnet"
|
||||
GENESIS_HASH = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
|
||||
@@ -37,7 +37,7 @@ class LBCDaemon(Daemon):
|
||||
@handles_errors
|
||||
async def getclaimsfortx(self, txid):
|
||||
'''Given a txid, returns the claims it make.'''
|
||||
return await self._send_single('getclaimsfortx', (txid,))
|
||||
return await self._send_single('getclaimsfortx', (txid,)) or []
|
||||
|
||||
@handles_errors
|
||||
async def getnameproof(self, name, block_hash=None):
|
||||
|
||||
@@ -21,7 +21,7 @@ class LBRYDB(DB):
|
||||
self.pending_abandons = {}
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def shutdown(self):
|
||||
def close(self):
|
||||
self.batched_flush_claims()
|
||||
self.claims_db.close()
|
||||
self.names_db.close()
|
||||
@@ -29,9 +29,7 @@ class LBRYDB(DB):
|
||||
self.outpoint_to_claim_id_db.close()
|
||||
self.claim_undo_db.close()
|
||||
self.utxo_db.close()
|
||||
# electrumx ones
|
||||
self.utxo_db.close()
|
||||
self.history.close_db()
|
||||
super().close()
|
||||
|
||||
async def _open_dbs(self, for_sync, compacting):
|
||||
await super()._open_dbs(for_sync=for_sync, compacting=compacting)
|
||||
|
||||
@@ -307,7 +307,7 @@ class LBRYElectrumX(ElectrumX):
|
||||
return {'error': err.message}
|
||||
result = {}
|
||||
|
||||
if parsed_uri.is_channel:
|
||||
if parsed_uri.contains_channel:
|
||||
certificate = None
|
||||
|
||||
# TODO: this is also done on the else, refactor
|
||||
|
||||
@@ -1,43 +1,11 @@
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import sys
|
||||
import traceback
|
||||
import treq
|
||||
import twisted.python.log
|
||||
from twisted.internet import defer
|
||||
from lbrynet import __version__ as lbrynet_version, build_type
|
||||
from lbrynet import utils, conf
|
||||
|
||||
|
||||
class HTTPSHandler(logging.Handler):
|
||||
def __init__(self, url, fqdn=False, localname=None, facility=None, cookies=None):
|
||||
super().__init__()
|
||||
self.url = url
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.cookies = cookies or {}
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _emit(self, record):
|
||||
payload = self.format(record)
|
||||
response = yield treq.post(self.url, data=payload.encode(), cookies=self.cookies)
|
||||
self.cookies.update(response.cookies())
|
||||
|
||||
def emit(self, record):
|
||||
return self._emit(record)
|
||||
|
||||
|
||||
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
|
||||
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
|
||||
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
|
||||
|
||||
|
||||
def remove_handlers(log, handler_name):
|
||||
@@ -108,63 +76,6 @@ def configure_file_handler(file_name, **kwargs):
|
||||
return handler
|
||||
|
||||
|
||||
def get_loggly_url(token=None, version=None):
|
||||
token = token or utils.deobfuscate(conf.settings['LOGGLY_TOKEN'])
|
||||
version = version or lbrynet_version
|
||||
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
|
||||
|
||||
|
||||
def configure_loggly_handler():
|
||||
if build_type.BUILD == 'dev':
|
||||
return
|
||||
if not conf.settings['share_usage_data']:
|
||||
return
|
||||
level = logging.ERROR
|
||||
handler = get_loggly_handler(level=level, installation_id=conf.settings.installation_id,
|
||||
session_id=conf.settings.get_session_id())
|
||||
log = logging.getLogger("lbrynet")
|
||||
if handler.name:
|
||||
remove_handlers(log, handler.name)
|
||||
handler.setLevel(level)
|
||||
log.addHandler(handler)
|
||||
# need to reduce the logger's level down to the
|
||||
# handler's level or else the handler won't
|
||||
# get those messages
|
||||
if log.level > level:
|
||||
log.setLevel(level)
|
||||
|
||||
|
||||
def get_loggly_handler(level, installation_id, session_id):
|
||||
formatter = JsonFormatter(level=level, installation_id=installation_id, session_id=session_id)
|
||||
handler = HTTPSHandler(get_loggly_url())
|
||||
handler.setFormatter(formatter)
|
||||
handler.name = 'loggly'
|
||||
return handler
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
def failure(failure, log, msg, *args):
|
||||
"""Log a failure message from a deferred.
|
||||
|
||||
|
||||
+31
-36
@@ -1,8 +1,10 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from binascii import unhexlify
|
||||
from sqlite3 import IntegrityError
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.internet import defer
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.creator import BlobFileCreator
|
||||
|
||||
@@ -26,15 +28,13 @@ class DiskBlobManager:
|
||||
self.blobs = {}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
async def setup(self):
|
||||
if self._node_datastore is not None:
|
||||
raw_blob_hashes = yield self.storage.get_all_finished_blobs()
|
||||
raw_blob_hashes = await self.storage.get_all_finished_blobs()
|
||||
self._node_datastore.completed_blobs.update(raw_blob_hashes)
|
||||
defer.returnValue(True)
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(True)
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
def get_blob(self, blob_hash, length=None):
|
||||
"""Return a blob identified by blob_hash, which may be a new blob or a
|
||||
@@ -43,7 +43,7 @@ class DiskBlobManager:
|
||||
if length is not None and not isinstance(length, int):
|
||||
raise Exception("invalid length type: {} ({})".format(length, str(type(length))))
|
||||
if blob_hash in self.blobs:
|
||||
return defer.succeed(self.blobs[blob_hash])
|
||||
return self.blobs[blob_hash]
|
||||
return self._make_new_blob(blob_hash, length)
|
||||
|
||||
def get_blob_creator(self):
|
||||
@@ -53,13 +53,13 @@ class DiskBlobManager:
|
||||
log.debug('Making a new blob for %s', blob_hash)
|
||||
blob = BlobFile(self.blob_dir, blob_hash, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
return defer.succeed(blob)
|
||||
return blob
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def blob_completed(self, blob, should_announce=False, next_announce_time=None):
|
||||
yield self.storage.add_completed_blob(
|
||||
yield f2d(self.storage.add_completed_blob(
|
||||
blob.blob_hash, blob.length, next_announce_time, should_announce
|
||||
)
|
||||
))
|
||||
if self._node_datastore is not None:
|
||||
self._node_datastore.completed_blobs.add(unhexlify(blob.blob_hash))
|
||||
|
||||
@@ -67,14 +67,15 @@ class DiskBlobManager:
|
||||
return self._completed_blobs(blobhashes_to_check)
|
||||
|
||||
def count_should_announce_blobs(self):
|
||||
return self.storage.count_should_announce_blobs()
|
||||
return f2d(self.storage.count_should_announce_blobs())
|
||||
|
||||
def set_should_announce(self, blob_hash, should_announce):
|
||||
now = self.storage.clock.seconds()
|
||||
return self.storage.set_should_announce(blob_hash, now, should_announce)
|
||||
return f2d(self.storage.set_should_announce(
|
||||
blob_hash, asyncio.get_event_loop().time(), should_announce
|
||||
))
|
||||
|
||||
def get_should_announce(self, blob_hash):
|
||||
return self.storage.should_announce(blob_hash)
|
||||
return f2d(self.storage.should_announce(blob_hash))
|
||||
|
||||
def creator_finished(self, blob_creator, should_announce):
|
||||
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
@@ -89,7 +90,7 @@ class DiskBlobManager:
|
||||
return self.blob_completed(new_blob, should_announce)
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d = f2d(self._get_all_verified_blob_hashes())
|
||||
d.addCallback(self.completed_blobs)
|
||||
return d
|
||||
|
||||
@@ -105,35 +106,29 @@ class DiskBlobManager:
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
blob = yield self.get_blob(blob_hash)
|
||||
yield blob.delete()
|
||||
blob = self.get_blob(blob_hash)
|
||||
blob.delete()
|
||||
bh_to_delete_from_db.append(blob_hash)
|
||||
del self.blobs[blob_hash]
|
||||
except Exception as e:
|
||||
log.warning("Failed to delete blob file. Reason: %s", e)
|
||||
try:
|
||||
yield self.storage.delete_blobs_from_db(bh_to_delete_from_db)
|
||||
yield f2d(self.storage.delete_blobs_from_db(bh_to_delete_from_db))
|
||||
except IntegrityError as err:
|
||||
if str(err) != "FOREIGN KEY constraint failed":
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _completed_blobs(self, blobhashes_to_check):
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = yield defer.DeferredList([self.get_blob(b) for b in blobhashes_to_check])
|
||||
blob_hashes = [b.blob_hash for success, b in blobs if success and b.verified]
|
||||
defer.returnValue(blob_hashes)
|
||||
blobs = [self.get_blob(b) for b in blobhashes_to_check]
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.verified]
|
||||
return blob_hashes
|
||||
|
||||
def _get_all_verified_blob_hashes(self):
|
||||
d = self.storage.get_all_blob_hashes()
|
||||
|
||||
def get_verified_blobs(blobs):
|
||||
verified_blobs = []
|
||||
for blob_hash in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
|
||||
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
|
||||
return d
|
||||
async def _get_all_verified_blob_hashes(self):
|
||||
blobs = await self.storage.get_all_blob_hashes()
|
||||
verified_blobs = []
|
||||
for blob_hash in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
@@ -5,7 +5,7 @@ from twisted.internet import defer, task
|
||||
from twisted.internet.error import ConnectingCancelledError
|
||||
from twisted.web._newclient import ResponseNeverReceived
|
||||
|
||||
from lbrynet.utils import DeferredDict
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.Error import DownloadCanceledError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -104,9 +104,7 @@ class HTTPBlobDownloader:
|
||||
log.debug("trying to download stream from mirror (sd %s)", self.sd_hashes[0][:8])
|
||||
else:
|
||||
log.debug("trying to download %i blobs from mirror", len(self.blob_hashes))
|
||||
blobs = yield DeferredDict(
|
||||
{blob_hash: self.blob_manager.get_blob(blob_hash) for blob_hash in self.blob_hashes}
|
||||
)
|
||||
blobs = {blob_hash: self.blob_manager.get_blob(blob_hash) for blob_hash in self.blob_hashes}
|
||||
self.deferreds = [self.download_blob(blobs[blob_hash]) for blob_hash in self.blob_hashes]
|
||||
yield defer.DeferredList(self.deferreds)
|
||||
if self.retry and self.missing_blob_hashes:
|
||||
@@ -159,7 +157,7 @@ class HTTPBlobDownloader:
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _write_blob(self, writer, blob):
|
||||
response = yield self.client.get(url_for(choice(self.servers), blob.blob_hash))
|
||||
response = yield self.client.get(url_for('{}:{}'.format(*choice(self.servers)), blob.blob_hash))
|
||||
if response.code != 200:
|
||||
log.debug('Missing a blob: %s', blob.blob_hash)
|
||||
if blob.blob_hash in self.blob_hashes:
|
||||
@@ -175,7 +173,7 @@ class HTTPBlobDownloader:
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_stream(self, stream_hash, sd_hash):
|
||||
stream_crypt_blobs = yield self.blob_manager.storage.get_blobs_for_stream(stream_hash)
|
||||
stream_crypt_blobs = yield f2d(self.blob_manager.storage.get_blobs_for_stream(stream_hash))
|
||||
self.blob_hashes.extend([
|
||||
b.blob_hash for b in stream_crypt_blobs
|
||||
if b.blob_hash and b.blob_hash not in self.blob_hashes
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
from decimal import Decimal
|
||||
from lbrynet import conf
|
||||
from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy
|
||||
|
||||
|
||||
class BasePaymentRateManager:
|
||||
def __init__(self, rate=None, info_rate=None):
|
||||
self.min_blob_data_payment_rate = rate if rate is not None else conf.settings['data_rate']
|
||||
self.min_blob_info_payment_rate = (
|
||||
info_rate if info_rate is not None else conf.settings['min_info_rate'])
|
||||
def __init__(self, rate, info_rate):
|
||||
self.min_blob_data_payment_rate = rate
|
||||
self.min_blob_info_payment_rate = info_rate
|
||||
|
||||
|
||||
class PaymentRateManager:
|
||||
@@ -37,7 +35,7 @@ class PaymentRateManager:
|
||||
|
||||
|
||||
class NegotiatedPaymentRateManager:
|
||||
def __init__(self, base, availability_tracker, generous=None):
|
||||
def __init__(self, base, availability_tracker, generous):
|
||||
"""
|
||||
@param base: a BasePaymentRateManager
|
||||
@param availability_tracker: a BlobAvailabilityTracker
|
||||
@@ -48,10 +46,10 @@ class NegotiatedPaymentRateManager:
|
||||
self.min_blob_data_payment_rate = base.min_blob_data_payment_rate
|
||||
self.points_paid = 0.0
|
||||
self.blob_tracker = availability_tracker
|
||||
self.generous = generous if generous is not None else conf.settings['is_generous_host']
|
||||
self.strategy = get_default_strategy(self.blob_tracker,
|
||||
base_price=self.base.min_blob_data_payment_rate,
|
||||
is_generous=generous)
|
||||
self.generous = generous
|
||||
self.strategy = get_default_strategy(
|
||||
self.blob_tracker, self.base.min_blob_data_payment_rate, generous
|
||||
)
|
||||
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self.strategy.make_offer(peer, blobs)
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
from decimal import Decimal
|
||||
from lbrynet import conf
|
||||
|
||||
|
||||
def get_default_price_model(blob_tracker, base_price, **kwargs):
|
||||
@@ -26,9 +25,8 @@ class MeanAvailabilityWeightedPrice:
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, tracker, base_price=None, alpha=1.0):
|
||||
def __init__(self, tracker, base_price, alpha=1.0):
|
||||
self.blob_tracker = tracker
|
||||
base_price = base_price if base_price is not None else conf.settings['data_rate']
|
||||
self.base_price = Decimal(base_price)
|
||||
self.alpha = Decimal(alpha)
|
||||
|
||||
|
||||
@@ -2,8 +2,9 @@ import logging
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from twisted.internet import defer, threads, reactor
|
||||
from twisted.internet import defer, reactor
|
||||
|
||||
from lbrynet.extras.compat import d2f
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.p2p.BlobManager import DiskBlobManager
|
||||
from lbrynet.p2p.RateLimiter import DummyRateLimiter
|
||||
@@ -13,6 +14,7 @@ from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder
|
||||
from lbrynet.conf import Config
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -59,7 +61,8 @@ class SingleBlobDownloadManager:
|
||||
|
||||
|
||||
class SinglePeerDownloader:
|
||||
def __init__(self):
|
||||
def __init__(self, conf: Config):
|
||||
self.conf = conf
|
||||
self._payment_rate_manager = OnlyFreePaymentsManager()
|
||||
self._rate_limiter = DummyRateLimiter()
|
||||
self._wallet = None
|
||||
@@ -80,7 +83,7 @@ class SinglePeerDownloader:
|
||||
peer_finder = SinglePeerFinder(peer)
|
||||
requester = BlobRequester(blob_manager, peer_finder, self._payment_rate_manager,
|
||||
self._wallet, download_manager)
|
||||
downloader = StandaloneBlobDownloader(blob_hash, blob_manager, peer_finder,
|
||||
downloader = StandaloneBlobDownloader(self.conf, blob_hash, blob_manager, peer_finder,
|
||||
self._rate_limiter, self._payment_rate_manager,
|
||||
self._wallet, timeout=timeout)
|
||||
info_exchanger = self._wallet.get_info_exchanger()
|
||||
@@ -91,19 +94,16 @@ class SinglePeerDownloader:
|
||||
result = yield blob.callback
|
||||
if not result:
|
||||
log.debug("Failed to downloaded %s from %s", blob_hash[:16], peer.host)
|
||||
yield connection_manager.stop()
|
||||
yield connection_manager.stop()
|
||||
defer.returnValue(result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
|
||||
tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
|
||||
tmp_storage = SQLiteStorage(tmp_dir)
|
||||
yield tmp_storage.setup()
|
||||
async def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
|
||||
tmp_storage = SQLiteStorage(Config(), ':memory:')
|
||||
await tmp_storage.open()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
tmp_blob_manager = DiskBlobManager(tmp_dir, tmp_storage)
|
||||
try:
|
||||
result = yield self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager)
|
||||
return await d2f(self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager))
|
||||
finally:
|
||||
yield tmp_blob_manager.stop()
|
||||
yield tmp_storage.stop()
|
||||
yield threads.deferToThread(shutil.rmtree, tmp_dir)
|
||||
defer.returnValue(result)
|
||||
await tmp_storage.close()
|
||||
shutil.rmtree(tmp_dir)
|
||||
+8
-12
@@ -1,11 +1,10 @@
|
||||
from decimal import Decimal
|
||||
from lbrynet import conf
|
||||
from lbrynet.p2p.Offer import Offer
|
||||
from lbrynet.p2p.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
|
||||
|
||||
|
||||
def get_default_strategy(blob_tracker, **kwargs):
|
||||
return BasicAvailabilityWeightedStrategy(blob_tracker, **kwargs)
|
||||
def get_default_strategy(blob_tracker, base_price, is_generous, **kwargs):
|
||||
return BasicAvailabilityWeightedStrategy(blob_tracker, base_price, is_generous, **kwargs)
|
||||
|
||||
|
||||
class Strategy:
|
||||
@@ -13,10 +12,9 @@ class Strategy:
|
||||
Base for negotiation strategies
|
||||
"""
|
||||
|
||||
def __init__(self, price_model, max_rate, min_rate, is_generous=None):
|
||||
def __init__(self, price_model, max_rate, min_rate, is_generous):
|
||||
self.price_model = price_model
|
||||
self.is_generous = (
|
||||
is_generous if is_generous is not None else conf.settings['is_generous_host'])
|
||||
self.is_generous = is_generous
|
||||
self.accepted_offers = {}
|
||||
self.pending_sent_offers = {}
|
||||
self.offers_sent = {}
|
||||
@@ -98,13 +96,11 @@ class BasicAvailabilityWeightedStrategy(Strategy):
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, blob_tracker, acceleration=1.25,
|
||||
deceleration=0.9, max_rate=None,
|
||||
min_rate=0.0,
|
||||
is_generous=None,
|
||||
base_price=0.0001, alpha=1.0):
|
||||
def __init__(self, blob_tracker, base_price, is_generous,
|
||||
acceleration=1.25, deceleration=0.9, max_rate=None,
|
||||
min_rate=0.0, alpha=1.0):
|
||||
price_model = MeanAvailabilityWeightedPrice(
|
||||
blob_tracker, base_price=base_price, alpha=alpha)
|
||||
blob_tracker, base_price, alpha=alpha)
|
||||
super().__init__(price_model, max_rate, min_rate, is_generous)
|
||||
self._acceleration = Decimal(acceleration) # rate of how quickly to ramp offer
|
||||
self._deceleration = Decimal(deceleration)
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections import defaultdict
|
||||
from binascii import unhexlify
|
||||
from twisted.internet import threads, defer
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.p2p.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
|
||||
@@ -260,7 +261,7 @@ def save_sd_info(blob_manager, sd_hash, sd_info):
|
||||
if calculated_sd_hash != sd_hash:
|
||||
raise InvalidStreamDescriptorError("%s does not match calculated %s" %
|
||||
(sd_hash, calculated_sd_hash))
|
||||
stream_hash = yield blob_manager.storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
stream_hash = yield f2d(blob_manager.storage.get_stream_hash_for_sd_hash(sd_hash))
|
||||
if not stream_hash:
|
||||
log.debug("Saving info for %s", unhexlify(sd_info['stream_name']))
|
||||
stream_name = sd_info['stream_name']
|
||||
@@ -268,10 +269,10 @@ def save_sd_info(blob_manager, sd_hash, sd_info):
|
||||
stream_hash = sd_info['stream_hash']
|
||||
stream_blobs = sd_info['blobs']
|
||||
suggested_file_name = sd_info['suggested_file_name']
|
||||
yield blob_manager.storage.add_known_blobs(stream_blobs)
|
||||
yield blob_manager.storage.store_stream(
|
||||
yield f2d(blob_manager.storage.add_known_blobs(stream_blobs))
|
||||
yield f2d(blob_manager.storage.store_stream(
|
||||
stream_hash, sd_hash, stream_name, key, suggested_file_name, stream_blobs
|
||||
)
|
||||
))
|
||||
defer.returnValue(stream_hash)
|
||||
|
||||
|
||||
@@ -299,8 +300,7 @@ def format_sd_info(stream_type, stream_name, key, suggested_file_name, stream_ha
|
||||
}
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_sd_info(storage, stream_hash, include_blobs):
|
||||
async def get_sd_info(storage, stream_hash, include_blobs):
|
||||
"""
|
||||
Get an sd info dictionary from storage
|
||||
|
||||
@@ -329,16 +329,13 @@ def get_sd_info(storage, stream_hash, include_blobs):
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
stream_info = yield storage.get_stream_info(stream_hash)
|
||||
stream_info = await storage.get_stream_info(stream_hash)
|
||||
blobs = []
|
||||
if include_blobs:
|
||||
blobs = yield storage.get_blobs_for_stream(stream_hash)
|
||||
defer.returnValue(
|
||||
format_sd_info(
|
||||
EncryptedFileStreamType, stream_info[0], stream_info[1],
|
||||
stream_info[2], stream_hash, format_blobs(blobs)
|
||||
)
|
||||
blobs = await storage.get_blobs_for_stream(stream_hash)
|
||||
return format_sd_info(
|
||||
EncryptedFileStreamType, stream_info[0], stream_info[1],
|
||||
stream_info[2], stream_hash, format_blobs(blobs)
|
||||
)
|
||||
|
||||
|
||||
@@ -432,8 +429,8 @@ class EncryptedFileStreamDescriptorValidator:
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_sd_blob(blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet, timeout=None,
|
||||
download_mirrors=None):
|
||||
def download_sd_blob(conf, blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet,
|
||||
timeout=None, download_mirrors=None):
|
||||
"""
|
||||
Downloads a single blob from the network
|
||||
|
||||
@@ -446,7 +443,8 @@ def download_sd_blob(blob_hash, blob_manager, peer_finder, rate_limiter, payment
|
||||
@return: An object of type HashBlob
|
||||
"""
|
||||
|
||||
downloader = StandaloneBlobDownloader(blob_hash,
|
||||
downloader = StandaloneBlobDownloader(conf,
|
||||
blob_hash,
|
||||
blob_manager,
|
||||
peer_finder,
|
||||
rate_limiter,
|
||||
@@ -465,6 +463,6 @@ def download_sd_blob(blob_hash, blob_manager, peer_finder, rate_limiter, payment
|
||||
yield blob_manager.delete_blobs([blob_hash])
|
||||
raise err
|
||||
raw_sd = yield sd_reader._get_raw_data()
|
||||
yield blob_manager.storage.add_known_blob(blob_hash, len(raw_sd))
|
||||
yield f2d(blob_manager.storage.add_known_blob(blob_hash, len(raw_sd)))
|
||||
yield save_sd_info(blob_manager, sd_blob.blob_hash, sd_info)
|
||||
defer.returnValue(sd_blob)
|
||||
@@ -5,7 +5,8 @@ from twisted.internet import error, defer
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.protocols.policies import TimeoutMixin
|
||||
from twisted.python import failure
|
||||
from lbrynet import conf, utils
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import MAX_RESPONSE_INFO_SIZE
|
||||
from lbrynet.p2p.Error import ConnectionClosedBeforeResponseError, NoResponseError
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, MisbehavingPeerError
|
||||
from lbrynet.p2p.Error import RequestCanceledError
|
||||
@@ -52,7 +53,7 @@ class ClientProtocol(Protocol, TimeoutMixin):
|
||||
self._blob_download_request.write(data)
|
||||
else:
|
||||
self._response_buff += data
|
||||
if len(self._response_buff) > conf.settings['MAX_RESPONSE_INFO_SIZE']:
|
||||
if len(self._response_buff) > MAX_RESPONSE_INFO_SIZE:
|
||||
log.warning("Response is too large from %s. Size %s",
|
||||
self.peer, len(self._response_buff))
|
||||
self.transport.loseConnection()
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import random
|
||||
import logging
|
||||
from twisted.internet import defer, reactor
|
||||
from lbrynet import utils, conf
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.p2p.client.ClientProtocol import ClientProtocolFactory
|
||||
from lbrynet.p2p.Error import InsufficientFundsError
|
||||
|
||||
@@ -20,11 +21,11 @@ class ConnectionManager:
|
||||
MANAGE_CALL_INTERVAL_SEC = 5
|
||||
TCP_CONNECT_TIMEOUT = 15
|
||||
|
||||
def __init__(self, downloader, rate_limiter,
|
||||
primary_request_creators, secondary_request_creators):
|
||||
def __init__(self, downloader, rate_limiter, primary_request_creators, secondary_request_creators):
|
||||
|
||||
self.seek_head_blob_first = conf.settings['seek_head_blob_first']
|
||||
self.max_connections_per_stream = conf.settings['max_connections_per_stream']
|
||||
self.conf: Config = downloader.conf
|
||||
self.seek_head_blob_first = self.conf.seek_head_blob_first
|
||||
self.max_connections_per_stream = self.conf.max_connections_per_stream
|
||||
|
||||
self.downloader = downloader
|
||||
self.rate_limiter = rate_limiter
|
||||
|
||||
@@ -30,7 +30,7 @@ class DownloadManager:
|
||||
def resume_downloading(self):
|
||||
yield self.connection_manager.start()
|
||||
yield self.progress_manager.start()
|
||||
defer.returnValue(True)
|
||||
return True
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop_downloading(self):
|
||||
@@ -39,32 +39,14 @@ class DownloadManager:
|
||||
defer.returnValue(True)
|
||||
|
||||
def add_blobs_to_download(self, blob_infos):
|
||||
|
||||
log.debug("Adding %s blobs to blobs", len(blob_infos))
|
||||
|
||||
def add_blob_to_list(blob, blob_num):
|
||||
self.blobs[blob_num] = blob
|
||||
log.debug(
|
||||
"Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_num)
|
||||
|
||||
def error_during_add(err):
|
||||
log.warning(
|
||||
"An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
ds = []
|
||||
for blob_info in blob_infos:
|
||||
if not blob_info.blob_num in self.blobs:
|
||||
self.blob_infos[blob_info.blob_num] = blob_info
|
||||
log.debug(
|
||||
"Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
d.addCallback(add_blob_to_list, blob_info.blob_num)
|
||||
d.addErrback(error_during_add)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
return dl
|
||||
log.debug("Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
|
||||
blob = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
self.blobs[blob_info.blob_num] = blob
|
||||
log.debug("Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_info.blob_num)
|
||||
|
||||
def stream_position(self):
|
||||
return self.progress_manager.stream_position()
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.p2p.BlobInfo import BlobInfo
|
||||
from lbrynet.p2p.client.BlobRequester import BlobRequester
|
||||
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
|
||||
@@ -82,9 +83,10 @@ class DummyBlobHandler:
|
||||
|
||||
|
||||
class StandaloneBlobDownloader:
|
||||
def __init__(self, blob_hash, blob_manager, peer_finder,
|
||||
def __init__(self, conf: Config, blob_hash, blob_manager, peer_finder,
|
||||
rate_limiter, payment_rate_manager, wallet,
|
||||
timeout=None):
|
||||
self.conf = conf
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_finder = peer_finder
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -39,19 +38,12 @@ class BlobAvailabilityHandler:
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
log.info("Received the client's list of requested blobs")
|
||||
d = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
|
||||
def set_field(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
|
||||
d.addCallback(set_field)
|
||||
return d
|
||||
return defer.succeed({})
|
||||
available_blobs = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
return {}
|
||||
|
||||
######### internal #########
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
|
||||
return d
|
||||
return self.blob_manager.completed_blobs(requested_blobs)
|
||||
@@ -61,21 +61,21 @@ class BlobRequestHandler:
|
||||
request_handler.register_blob_sender(self)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
response = defer.succeed({})
|
||||
response = {}
|
||||
log.debug("Handle query: %s", str(queries))
|
||||
|
||||
if self.AVAILABILITY_QUERY in queries:
|
||||
self._blobs_requested = queries[self.AVAILABILITY_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_availability(r, self._blobs_requested))
|
||||
self._reply_to_availability(response, self._blobs_requested)
|
||||
if self.PAYMENT_RATE_QUERY in queries:
|
||||
offered_rate = queries[self.PAYMENT_RATE_QUERY]
|
||||
offer = Offer(offered_rate)
|
||||
if offer.rate is None:
|
||||
log.warning("Empty rate offer")
|
||||
response.addCallback(lambda r: self._handle_payment_rate_query(offer, r))
|
||||
self._handle_payment_rate_query(offer, response)
|
||||
if self.BLOB_QUERY in queries:
|
||||
incoming = queries[self.BLOB_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_send_request(r, incoming))
|
||||
self._reply_to_send_request(response, incoming)
|
||||
return response
|
||||
|
||||
######### IBlobSender #########
|
||||
@@ -95,15 +95,10 @@ class BlobRequestHandler:
|
||||
######### internal #########
|
||||
|
||||
def _reply_to_availability(self, request, blobs):
|
||||
d = self._get_available_blobs(blobs)
|
||||
|
||||
def set_available(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
request.update({'available_blobs': available_blobs})
|
||||
return request
|
||||
|
||||
d.addCallback(set_available)
|
||||
return d
|
||||
available_blobs = self._get_available_blobs(blobs)
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
request.update({'available_blobs': available_blobs})
|
||||
return request
|
||||
|
||||
def _handle_payment_rate_query(self, offer, request):
|
||||
blobs = self._blobs_requested
|
||||
@@ -172,8 +167,7 @@ class BlobRequestHandler:
|
||||
return d
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
return d
|
||||
return self.blob_manager.completed_blobs(requested_blobs)
|
||||
|
||||
def send_file(self, consumer):
|
||||
|
||||
|
||||
@@ -167,7 +167,7 @@ class ServerRequestHandler:
|
||||
ds = []
|
||||
for query_handler, query_identifiers in self.query_handlers.items():
|
||||
queries = {q_i: msg[q_i] for q_i in query_identifiers if q_i in msg}
|
||||
d = query_handler.handle_queries(queries)
|
||||
d = defer.maybeDeferred(query_handler.handle_queries(queries))
|
||||
d.addErrback(log_errors)
|
||||
ds.append(d)
|
||||
|
||||
|
||||
+30
-25
@@ -7,19 +7,21 @@ from collections import OrderedDict
|
||||
|
||||
from lbrynet.schema.schema.claim import Claim
|
||||
from lbrynet.schema.proto import claim_pb2
|
||||
from lbrynet.schema.signature import Signature
|
||||
from lbrynet.schema.validator import get_validator
|
||||
from lbrynet.schema.signer import get_signer
|
||||
from lbrynet.schema.schema import NIST256p, CURVE_NAMES, CLAIM_TYPE_NAMES
|
||||
from lbrynet.schema.schema import NIST256p, CURVE_NAMES, CLAIM_TYPE_NAMES, SECP256k1
|
||||
from lbrynet.schema.encoding import decode_fields, decode_b64_fields, encode_fields
|
||||
from lbrynet.schema.error import DecodeError
|
||||
from lbrynet.schema.fee import Fee
|
||||
|
||||
|
||||
class ClaimDict(OrderedDict):
|
||||
def __init__(self, claim_dict=None):
|
||||
def __init__(self, claim_dict=None, detached_signature: Signature=None):
|
||||
if isinstance(claim_dict, claim_pb2.Claim):
|
||||
raise Exception("To initialize %s with a Claim protobuf use %s.load_protobuf" %
|
||||
(self.__class__.__name__, self.__class__.__name__))
|
||||
self.detached_signature = detached_signature
|
||||
OrderedDict.__init__(self, claim_dict or [])
|
||||
|
||||
@property
|
||||
@@ -37,7 +39,8 @@ class ClaimDict(OrderedDict):
|
||||
@property
|
||||
def serialized(self):
|
||||
"""Serialized Claim protobuf"""
|
||||
|
||||
if self.detached_signature:
|
||||
return self.detached_signature.serialized
|
||||
return self.protobuf.SerializeToString()
|
||||
|
||||
@property
|
||||
@@ -49,10 +52,9 @@ class ClaimDict(OrderedDict):
|
||||
|
||||
@property
|
||||
def has_signature(self):
|
||||
claim = self.protobuf
|
||||
if claim.HasField("publisherSignature"):
|
||||
return True
|
||||
return False
|
||||
return self.protobuf.HasField("publisherSignature") or (
|
||||
self.detached_signature and self.detached_signature.raw_signature
|
||||
)
|
||||
|
||||
@property
|
||||
def is_certificate(self):
|
||||
@@ -90,10 +92,11 @@ class ClaimDict(OrderedDict):
|
||||
return None
|
||||
|
||||
@property
|
||||
def certificate_id(self):
|
||||
if not self.has_signature:
|
||||
return None
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.certificateId)
|
||||
def certificate_id(self) -> str:
|
||||
if self.protobuf.HasField("publisherSignature"):
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.certificateId).decode()
|
||||
if self.detached_signature and self.detached_signature.certificate_id:
|
||||
return binascii.hexlify(self.detached_signature.certificate_id).decode()
|
||||
|
||||
@property
|
||||
def signature(self):
|
||||
@@ -117,57 +120,59 @@ class ClaimDict(OrderedDict):
|
||||
def claim_dict(self):
|
||||
"""Claim dictionary with bytes represented as hex and base58"""
|
||||
|
||||
return dict(encode_fields(self))
|
||||
return dict(encode_fields(self, self.detached_signature))
|
||||
|
||||
@classmethod
|
||||
def load_protobuf_dict(cls, protobuf_dict):
|
||||
def load_protobuf_dict(cls, protobuf_dict, detached_signature=None):
|
||||
"""
|
||||
Load a ClaimDict from a dictionary with base64 encoded bytes
|
||||
(as returned by the protobuf json formatter)
|
||||
"""
|
||||
|
||||
return cls(decode_b64_fields(protobuf_dict))
|
||||
return cls(decode_b64_fields(protobuf_dict), detached_signature=detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_protobuf(cls, protobuf_claim):
|
||||
def load_protobuf(cls, protobuf_claim, detached_signature=None):
|
||||
"""Load ClaimDict from a protobuf Claim message"""
|
||||
return cls.load_protobuf_dict(json.loads(json_format.MessageToJson(protobuf_claim, True)))
|
||||
return cls.load_protobuf_dict(json.loads(json_format.MessageToJson(protobuf_claim, True)), detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_dict(cls, claim_dict):
|
||||
"""Load ClaimDict from a dictionary with hex and base58 encoded bytes"""
|
||||
try:
|
||||
return cls.load_protobuf(cls(decode_fields(claim_dict)).protobuf)
|
||||
claim_dict, detached_signature = decode_fields(claim_dict)
|
||||
return cls.load_protobuf(cls(claim_dict).protobuf, detached_signature)
|
||||
except json_format.ParseError as err:
|
||||
raise DecodeError(str(err))
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, serialized):
|
||||
"""Load a ClaimDict from a serialized protobuf string"""
|
||||
detached_signature = Signature.flagged_parse(serialized)
|
||||
|
||||
temp_claim = claim_pb2.Claim()
|
||||
try:
|
||||
temp_claim.ParseFromString(serialized)
|
||||
temp_claim.ParseFromString(detached_signature.payload)
|
||||
except DecodeError_pb:
|
||||
raise DecodeError(DecodeError_pb)
|
||||
return cls.load_protobuf(temp_claim)
|
||||
return cls.load_protobuf(temp_claim, detached_signature=detached_signature)
|
||||
|
||||
@classmethod
|
||||
def generate_certificate(cls, private_key, curve=NIST256p):
|
||||
def generate_certificate(cls, private_key, curve=SECP256k1):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
return cls.load_protobuf(signer.certificate)
|
||||
|
||||
def sign(self, private_key, claim_address, cert_claim_id, curve=NIST256p):
|
||||
def sign(self, private_key, claim_address, cert_claim_id, curve=SECP256k1, name=None, legacy=False):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
signed = signer.sign_stream_claim(self, claim_address, cert_claim_id)
|
||||
return ClaimDict.load_protobuf(signed)
|
||||
signed, signature = signer.sign_stream_claim(self, claim_address, cert_claim_id, name, legacy)
|
||||
return ClaimDict.load_protobuf(signed, signature)
|
||||
|
||||
def validate_signature(self, claim_address, certificate):
|
||||
def validate_signature(self, claim_address, certificate, name=None):
|
||||
if isinstance(certificate, ClaimDict):
|
||||
certificate = certificate.protobuf
|
||||
curve = CURVE_NAMES[certificate.certificate.keyType]
|
||||
validator = get_validator(curve).load_from_certificate(certificate, self.certificate_id)
|
||||
return validator.validate_claim_signature(self, claim_address)
|
||||
return validator.validate_claim_signature(self, claim_address, name)
|
||||
|
||||
def validate_private_key(self, private_key, certificate_id):
|
||||
certificate = self.protobuf
|
||||
|
||||
@@ -4,15 +4,20 @@ from lbrynet.schema.address import decode_address, encode_address
|
||||
from lbrynet.schema.schema import CLAIM_TYPES, CLAIM_TYPE, STREAM_TYPE, CERTIFICATE_TYPE
|
||||
from lbrynet.schema.schema import SIGNATURE
|
||||
from lbrynet.schema.error import DecodeError, InvalidAddress
|
||||
from lbrynet.schema.signature import Signature
|
||||
|
||||
|
||||
def encode_fields(claim_dictionary):
|
||||
def encode_fields(claim_dictionary, detached_signature: Signature):
|
||||
"""Encode bytes to hex and b58 for return by ClaimDict"""
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
|
||||
claim_value['source']['source'] = binascii.hexlify(claim_value['source']['source']).decode()
|
||||
if 'releaseTime' in claim_value['metadata']:
|
||||
release_time = int(claim_value['metadata'].pop('releaseTime'))
|
||||
if release_time != 0:
|
||||
claim_value['metadata']['releaseTime'] = release_time
|
||||
if 'fee' in claim_value['metadata']:
|
||||
try:
|
||||
address = encode_address(claim_value['metadata']['fee']['address'])
|
||||
@@ -27,12 +32,19 @@ def encode_fields(claim_dictionary):
|
||||
encoded_cert_id = binascii.hexlify(claim_dictionary[SIGNATURE]['certificateId']).decode()
|
||||
claim_dictionary[SIGNATURE]['signature'] = encoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = encoded_cert_id
|
||||
elif detached_signature and detached_signature.raw_signature:
|
||||
claim_dictionary[SIGNATURE] = {
|
||||
'detached_signature': binascii.hexlify(detached_signature.serialized).decode(),
|
||||
'certificateId': binascii.hexlify(detached_signature.certificate_id).decode()
|
||||
}
|
||||
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
|
||||
|
||||
def decode_fields(claim_dictionary):
|
||||
"""Decode hex and b58 encoded bytes in dictionaries given to ClaimDict"""
|
||||
detached_signature = None
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
@@ -47,13 +59,17 @@ def decode_fields(claim_dictionary):
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = binascii.unhexlify(claim_value["publicKey"])
|
||||
claim_value["publicKey"] = public_key
|
||||
if SIGNATURE in claim_dictionary:
|
||||
if SIGNATURE in claim_dictionary and not claim_dictionary[SIGNATURE].get('detached_signature'):
|
||||
decoded_sig = binascii.unhexlify(claim_dictionary[SIGNATURE]['signature'])
|
||||
decoded_cert_id = binascii.unhexlify(claim_dictionary[SIGNATURE]['certificateId'])
|
||||
claim_dictionary[SIGNATURE]['signature'] = decoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = decoded_cert_id
|
||||
elif claim_dictionary.get(SIGNATURE, {}).get('detached_signature'):
|
||||
hex_detached_signature = claim_dictionary[SIGNATURE]['detached_signature']
|
||||
detached_signature = Signature.flagged_parse(binascii.unhexlify(hex_detached_signature))
|
||||
del claim_dictionary[SIGNATURE]
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
return claim_dictionary, detached_signature
|
||||
|
||||
|
||||
def decode_b64_fields(claim_dictionary):
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
enum KeyType {
|
||||
UNKNOWN_PUBLIC_KEY_TYPE = 0;
|
||||
NIST256p = 1;
|
||||
NIST384p = 2;
|
||||
SECP256k1 = 3;
|
||||
}
|
||||
|
||||
message Certificate {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required KeyType keyType = 2;
|
||||
required bytes publicKey = 4;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/certificate.proto
|
||||
# source: certificate.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -16,16 +16,16 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/certificate.proto',
|
||||
package='',
|
||||
name='certificate.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n&lbrynet/schema/proto/certificate.proto\"\x8e\x01\n\x0b\x43\x65rtificate\x12%\n\x07version\x18\x01 \x02(\x0e\x32\x14.Certificate.Version\x12\x19\n\x07keyType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
|
||||
serialized_pb=_b('\n\x11\x63\x65rtificate.proto\x12\x02pb\"\x94\x01\n\x0b\x43\x65rtificate\x12(\n\x07version\x18\x01 \x02(\x0e\x32\x17.pb.Certificate.Version\x12\x1c\n\x07keyType\x18\x02 \x02(\x0e\x32\x0b.pb.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
|
||||
)
|
||||
|
||||
_KEYTYPE = _descriptor.EnumDescriptor(
|
||||
name='KeyType',
|
||||
full_name='KeyType',
|
||||
full_name='pb.KeyType',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -48,8 +48,8 @@ _KEYTYPE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=187,
|
||||
serialized_end=268,
|
||||
serialized_start=176,
|
||||
serialized_end=257,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_KEYTYPE)
|
||||
|
||||
@@ -62,7 +62,7 @@ SECP256k1 = 3
|
||||
|
||||
_CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Certificate.Version',
|
||||
full_name='pb.Certificate.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -77,35 +77,35 @@ _CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=143,
|
||||
serialized_end=185,
|
||||
serialized_start=132,
|
||||
serialized_end=174,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CERTIFICATE_VERSION)
|
||||
|
||||
|
||||
_CERTIFICATE = _descriptor.Descriptor(
|
||||
name='Certificate',
|
||||
full_name='Certificate',
|
||||
full_name='pb.Certificate',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Certificate.version', index=0,
|
||||
name='version', full_name='pb.Certificate.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='keyType', full_name='Certificate.keyType', index=1,
|
||||
name='keyType', full_name='pb.Certificate.keyType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='publicKey', full_name='Certificate.publicKey', index=2,
|
||||
name='publicKey', full_name='pb.Certificate.publicKey', index=2,
|
||||
number=4, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -124,8 +124,8 @@ _CERTIFICATE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=43,
|
||||
serialized_end=185,
|
||||
serialized_start=26,
|
||||
serialized_end=174,
|
||||
)
|
||||
|
||||
_CERTIFICATE.fields_by_name['version'].enum_type = _CERTIFICATE_VERSION
|
||||
@@ -137,8 +137,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Certificate = _reflection.GeneratedProtocolMessageType('Certificate', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CERTIFICATE,
|
||||
__module__ = 'lbrynet.schema.proto.certificate_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Certificate)
|
||||
__module__ = 'certificate_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Certificate)
|
||||
))
|
||||
_sym_db.RegisterMessage(Certificate)
|
||||
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/stream.proto";
|
||||
import "lbrynet/schema/proto/certificate.proto";
|
||||
import "lbrynet/schema/proto/signature.proto";
|
||||
|
||||
message Claim {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
enum ClaimType {
|
||||
UNKNOWN_CLAIM_TYPE = 0;
|
||||
streamType = 1;
|
||||
certificateType = 2;
|
||||
}
|
||||
required ClaimType claimType = 2;
|
||||
optional Stream stream = 3;
|
||||
optional Certificate certificate = 4;
|
||||
optional Signature publisherSignature = 5;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/claim.proto
|
||||
# source: claim.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,25 +12,25 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import stream_pb2 as lbrynet_dot_schema_dot_proto_dot_stream__pb2
|
||||
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
|
||||
from lbrynet.schema.proto import signature_pb2 as lbrynet_dot_schema_dot_proto_dot_signature__pb2
|
||||
from . import stream_pb2 as stream__pb2
|
||||
from . import certificate_pb2 as certificate__pb2
|
||||
from . import signature_pb2 as signature__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/claim.proto',
|
||||
package='',
|
||||
name='claim.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n lbrynet/schema/proto/claim.proto\x1a!lbrynet/schema/proto/stream.proto\x1a&lbrynet/schema/proto/certificate.proto\x1a$lbrynet/schema/proto/signature.proto\"\xa7\x02\n\x05\x43laim\x12\x1f\n\x07version\x18\x01 \x02(\x0e\x32\x0e.Claim.Version\x12#\n\tclaimType\x18\x02 \x02(\x0e\x32\x10.Claim.ClaimType\x12\x17\n\x06stream\x18\x03 \x01(\x0b\x32\x07.Stream\x12!\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x0c.Certificate\x12&\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\n.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
|
||||
serialized_pb=_b('\n\x0b\x63laim.proto\x12\x02pb\x1a\x0cstream.proto\x1a\x11\x63\x65rtificate.proto\x1a\x0fsignature.proto\"\xb6\x02\n\x05\x43laim\x12\"\n\x07version\x18\x01 \x02(\x0e\x32\x11.pb.Claim.Version\x12&\n\tclaimType\x18\x02 \x02(\x0e\x32\x13.pb.Claim.ClaimType\x12\x1a\n\x06stream\x18\x03 \x01(\x0b\x32\n.pb.Stream\x12$\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x0f.pb.Certificate\x12)\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\r.pb.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_stream__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_signature__pb2.DESCRIPTOR,])
|
||||
dependencies=[stream__pb2.DESCRIPTOR,certificate__pb2.DESCRIPTOR,signature__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_CLAIM_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Claim.Version',
|
||||
full_name='pb.Claim.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -45,14 +45,14 @@ _CLAIM_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=329,
|
||||
serialized_end=371,
|
||||
serialized_start=264,
|
||||
serialized_end=306,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CLAIM_VERSION)
|
||||
|
||||
_CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
|
||||
name='ClaimType',
|
||||
full_name='Claim.ClaimType',
|
||||
full_name='pb.Claim.ClaimType',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -71,49 +71,49 @@ _CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=373,
|
||||
serialized_end=445,
|
||||
serialized_start=308,
|
||||
serialized_end=380,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CLAIM_CLAIMTYPE)
|
||||
|
||||
|
||||
_CLAIM = _descriptor.Descriptor(
|
||||
name='Claim',
|
||||
full_name='Claim',
|
||||
full_name='pb.Claim',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Claim.version', index=0,
|
||||
name='version', full_name='pb.Claim.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claimType', full_name='Claim.claimType', index=1,
|
||||
name='claimType', full_name='pb.Claim.claimType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='stream', full_name='Claim.stream', index=2,
|
||||
name='stream', full_name='pb.Claim.stream', index=2,
|
||||
number=3, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='certificate', full_name='Claim.certificate', index=3,
|
||||
name='certificate', full_name='pb.Claim.certificate', index=3,
|
||||
number=4, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='publisherSignature', full_name='Claim.publisherSignature', index=4,
|
||||
name='publisherSignature', full_name='pb.Claim.publisherSignature', index=4,
|
||||
number=5, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -133,15 +133,15 @@ _CLAIM = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=150,
|
||||
serialized_end=445,
|
||||
serialized_start=70,
|
||||
serialized_end=380,
|
||||
)
|
||||
|
||||
_CLAIM.fields_by_name['version'].enum_type = _CLAIM_VERSION
|
||||
_CLAIM.fields_by_name['claimType'].enum_type = _CLAIM_CLAIMTYPE
|
||||
_CLAIM.fields_by_name['stream'].message_type = lbrynet_dot_schema_dot_proto_dot_stream__pb2._STREAM
|
||||
_CLAIM.fields_by_name['certificate'].message_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._CERTIFICATE
|
||||
_CLAIM.fields_by_name['publisherSignature'].message_type = lbrynet_dot_schema_dot_proto_dot_signature__pb2._SIGNATURE
|
||||
_CLAIM.fields_by_name['stream'].message_type = stream__pb2._STREAM
|
||||
_CLAIM.fields_by_name['certificate'].message_type = certificate__pb2._CERTIFICATE
|
||||
_CLAIM.fields_by_name['publisherSignature'].message_type = signature__pb2._SIGNATURE
|
||||
_CLAIM_VERSION.containing_type = _CLAIM
|
||||
_CLAIM_CLAIMTYPE.containing_type = _CLAIM
|
||||
DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
|
||||
@@ -149,8 +149,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CLAIM,
|
||||
__module__ = 'lbrynet.schema.proto.claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Claim)
|
||||
__module__ = 'claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Claim)
|
||||
))
|
||||
_sym_db.RegisterMessage(Claim)
|
||||
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
message Fee {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
enum Currency {
|
||||
UNKNOWN_CURRENCY = 0;
|
||||
LBC = 1;
|
||||
BTC = 2;
|
||||
USD = 3;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Currency currency = 2;
|
||||
required bytes address = 3;
|
||||
required float amount = 4;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/fee.proto
|
||||
# source: fee.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/fee.proto',
|
||||
package='',
|
||||
name='fee.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x1elbrynet/schema/proto/fee.proto\"\xcf\x01\n\x03\x46\x65\x65\x12\x1d\n\x07version\x18\x01 \x02(\x0e\x32\x0c.Fee.Version\x12\x1f\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\r.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
|
||||
serialized_pb=_b('\n\tfee.proto\x12\x02pb\"\xd5\x01\n\x03\x46\x65\x65\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.pb.Fee.Version\x12\"\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\x10.pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
|
||||
)
|
||||
|
||||
|
||||
|
||||
_FEE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Fee.Version',
|
||||
full_name='pb.Fee.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -41,14 +41,14 @@ _FEE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=139,
|
||||
serialized_end=181,
|
||||
serialized_start=128,
|
||||
serialized_end=170,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_FEE_VERSION)
|
||||
|
||||
_FEE_CURRENCY = _descriptor.EnumDescriptor(
|
||||
name='Currency',
|
||||
full_name='Fee.Currency',
|
||||
full_name='pb.Fee.Currency',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -71,42 +71,42 @@ _FEE_CURRENCY = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=183,
|
||||
serialized_end=242,
|
||||
serialized_start=172,
|
||||
serialized_end=231,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
|
||||
|
||||
|
||||
_FEE = _descriptor.Descriptor(
|
||||
name='Fee',
|
||||
full_name='Fee',
|
||||
full_name='pb.Fee',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Fee.version', index=0,
|
||||
name='version', full_name='pb.Fee.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='currency', full_name='Fee.currency', index=1,
|
||||
name='currency', full_name='pb.Fee.currency', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='address', full_name='Fee.address', index=2,
|
||||
name='address', full_name='pb.Fee.address', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='amount', full_name='Fee.amount', index=3,
|
||||
name='amount', full_name='pb.Fee.amount', index=3,
|
||||
number=4, type=2, cpp_type=6, label=2,
|
||||
has_default_value=False, default_value=float(0),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -126,8 +126,8 @@ _FEE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=35,
|
||||
serialized_end=242,
|
||||
serialized_start=18,
|
||||
serialized_end=231,
|
||||
)
|
||||
|
||||
_FEE.fields_by_name['version'].enum_type = _FEE_VERSION
|
||||
@@ -139,8 +139,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
|
||||
DESCRIPTOR = _FEE,
|
||||
__module__ = 'lbrynet.schema.proto.fee_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Fee)
|
||||
__module__ = 'fee_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Fee)
|
||||
))
|
||||
_sym_db.RegisterMessage(Fee)
|
||||
|
||||
|
||||
@@ -1,212 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/fee.proto";
|
||||
|
||||
message Metadata {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
_0_0_2 = 2;
|
||||
_0_0_3 = 3;
|
||||
_0_1_0 = 4;
|
||||
}
|
||||
enum Language {
|
||||
UNKNOWN_LANGUAGE = 0;
|
||||
en = 1;
|
||||
aa = 2;
|
||||
ab = 3;
|
||||
ae = 4;
|
||||
af = 5;
|
||||
ak = 6;
|
||||
am = 7;
|
||||
an = 8;
|
||||
ar = 9;
|
||||
as = 10;
|
||||
av = 11;
|
||||
ay = 12;
|
||||
az = 13;
|
||||
ba = 14;
|
||||
be = 15;
|
||||
bg = 16;
|
||||
bh = 17;
|
||||
bi = 18;
|
||||
bm = 19;
|
||||
bn = 20;
|
||||
bo = 21;
|
||||
br = 22;
|
||||
bs = 23;
|
||||
ca = 24;
|
||||
ce = 25;
|
||||
ch = 26;
|
||||
co = 27;
|
||||
cr = 28;
|
||||
cs = 29;
|
||||
cu = 30;
|
||||
cv = 31;
|
||||
cy = 32;
|
||||
da = 33;
|
||||
de = 34;
|
||||
dv = 35;
|
||||
dz = 36;
|
||||
ee = 37;
|
||||
el = 38;
|
||||
eo = 39;
|
||||
es = 40;
|
||||
et = 41;
|
||||
eu = 42;
|
||||
fa = 43;
|
||||
ff = 44;
|
||||
fi = 45;
|
||||
fj = 46;
|
||||
fo = 47;
|
||||
fr = 48;
|
||||
fy = 49;
|
||||
ga = 50;
|
||||
gd = 51;
|
||||
gl = 52;
|
||||
gn = 53;
|
||||
gu = 54;
|
||||
gv = 55;
|
||||
ha = 56;
|
||||
he = 57;
|
||||
hi = 58;
|
||||
ho = 59;
|
||||
hr = 60;
|
||||
ht = 61;
|
||||
hu = 62;
|
||||
hy = 63;
|
||||
hz = 64;
|
||||
ia = 65;
|
||||
id = 66;
|
||||
ie = 67;
|
||||
ig = 68;
|
||||
ii = 69;
|
||||
ik = 70;
|
||||
io = 71;
|
||||
is = 72;
|
||||
it = 73;
|
||||
iu = 74;
|
||||
ja = 75;
|
||||
jv = 76;
|
||||
ka = 77;
|
||||
kg = 78;
|
||||
ki = 79;
|
||||
kj = 80;
|
||||
kk = 81;
|
||||
kl = 82;
|
||||
km = 83;
|
||||
kn = 84;
|
||||
ko = 85;
|
||||
kr = 86;
|
||||
ks = 87;
|
||||
ku = 88;
|
||||
kv = 89;
|
||||
kw = 90;
|
||||
ky = 91;
|
||||
la = 92;
|
||||
lb = 93;
|
||||
lg = 94;
|
||||
li = 95;
|
||||
ln = 96;
|
||||
lo = 97;
|
||||
lt = 98;
|
||||
lu = 99;
|
||||
lv = 100;
|
||||
mg = 101;
|
||||
mh = 102;
|
||||
mi = 103;
|
||||
mk = 104;
|
||||
ml = 105;
|
||||
mn = 106;
|
||||
mr = 107;
|
||||
ms = 108;
|
||||
mt = 109;
|
||||
my = 110;
|
||||
na = 111;
|
||||
nb = 112;
|
||||
nd = 113;
|
||||
ne = 114;
|
||||
ng = 115;
|
||||
nl = 116;
|
||||
nn = 117;
|
||||
no = 118;
|
||||
nr = 119;
|
||||
nv = 120;
|
||||
ny = 121;
|
||||
oc = 122;
|
||||
oj = 123;
|
||||
om = 124;
|
||||
or = 125;
|
||||
os = 126;
|
||||
pa = 127;
|
||||
pi = 128;
|
||||
pl = 129;
|
||||
ps = 130;
|
||||
pt = 131;
|
||||
qu = 132;
|
||||
rm = 133;
|
||||
rn = 134;
|
||||
ro = 135;
|
||||
ru = 136;
|
||||
rw = 137;
|
||||
sa = 138;
|
||||
sc = 139;
|
||||
sd = 140;
|
||||
se = 141;
|
||||
sg = 142;
|
||||
si = 143;
|
||||
sk = 144;
|
||||
sl = 145;
|
||||
sm = 146;
|
||||
sn = 147;
|
||||
so = 148;
|
||||
sq = 149;
|
||||
sr = 150;
|
||||
ss = 151;
|
||||
st = 152;
|
||||
su = 153;
|
||||
sv = 154;
|
||||
sw = 155;
|
||||
ta = 156;
|
||||
te = 157;
|
||||
tg = 158;
|
||||
th = 159;
|
||||
ti = 160;
|
||||
tk = 161;
|
||||
tl = 162;
|
||||
tn = 163;
|
||||
to = 164;
|
||||
tr = 165;
|
||||
ts = 166;
|
||||
tt = 167;
|
||||
tw = 168;
|
||||
ty = 169;
|
||||
ug = 170;
|
||||
uk = 171;
|
||||
ur = 172;
|
||||
uz = 173;
|
||||
ve = 174;
|
||||
vi = 175;
|
||||
vo = 176;
|
||||
wa = 177;
|
||||
wo = 178;
|
||||
xh = 179;
|
||||
yi = 180;
|
||||
yo = 181;
|
||||
za = 182;
|
||||
zh = 183;
|
||||
zu = 184;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Language language = 2;
|
||||
required string title = 3;
|
||||
required string description = 4;
|
||||
required string author = 5;
|
||||
required string license = 6;
|
||||
required bool nsfw = 7;
|
||||
|
||||
optional Fee fee = 8;
|
||||
optional string thumbnail = 9;
|
||||
optional string preview = 10;
|
||||
optional string licenseUrl = 11;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/metadata.proto
|
||||
# source: metadata.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import fee_pb2 as lbrynet_dot_schema_dot_proto_dot_fee__pb2
|
||||
from . import fee_pb2 as fee__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/metadata.proto',
|
||||
package='',
|
||||
name='metadata.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n#lbrynet/schema/proto/metadata.proto\x1a\x1elbrynet/schema/proto/fee.proto\"\xde\x0e\n\x08Metadata\x12\"\n\x07version\x18\x01 \x02(\x0e\x32\x11.Metadata.Version\x12$\n\x08language\x18\x02 \x02(\x0e\x32\x12.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x11\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x04.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02ty\x1Line truncated
|
||||
serialized_pb=_b('\n\x0emetadata.proto\x12\x02pb\x1a\tfee.proto\"\xfc\x0e\n\x08Metadata\x12%\n\x07version\x18\x01 \x02(\x0e\x32\x14.pb.Metadata.Version\x12\'\n\x08language\x18\x02 \x02(\x0e\x32\x15.pb.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x14\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x07.pb.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\x12\x13\n\x0breleaseTime\x18\x0c \x01(\x12\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\xLine truncated
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_fee__pb2.DESCRIPTOR,])
|
||||
dependencies=[fee__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_METADATA_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Metadata.Version',
|
||||
full_name='pb.Metadata.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -55,14 +55,14 @@ _METADATA_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=316,
|
||||
serialized_end=394,
|
||||
serialized_start=308,
|
||||
serialized_end=386,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_METADATA_VERSION)
|
||||
|
||||
_METADATA_LANGUAGE = _descriptor.EnumDescriptor(
|
||||
name='Language',
|
||||
full_name='Metadata.Language',
|
||||
full_name='pb.Metadata.Language',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -809,96 +809,103 @@ _METADATA_LANGUAGE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=397,
|
||||
serialized_end=1958,
|
||||
serialized_start=389,
|
||||
serialized_end=1950,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_METADATA_LANGUAGE)
|
||||
|
||||
|
||||
_METADATA = _descriptor.Descriptor(
|
||||
name='Metadata',
|
||||
full_name='Metadata',
|
||||
full_name='pb.Metadata',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Metadata.version', index=0,
|
||||
name='version', full_name='pb.Metadata.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='language', full_name='Metadata.language', index=1,
|
||||
name='language', full_name='pb.Metadata.language', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='title', full_name='Metadata.title', index=2,
|
||||
name='title', full_name='pb.Metadata.title', index=2,
|
||||
number=3, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='description', full_name='Metadata.description', index=3,
|
||||
name='description', full_name='pb.Metadata.description', index=3,
|
||||
number=4, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='author', full_name='Metadata.author', index=4,
|
||||
name='author', full_name='pb.Metadata.author', index=4,
|
||||
number=5, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='license', full_name='Metadata.license', index=5,
|
||||
name='license', full_name='pb.Metadata.license', index=5,
|
||||
number=6, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='nsfw', full_name='Metadata.nsfw', index=6,
|
||||
name='nsfw', full_name='pb.Metadata.nsfw', index=6,
|
||||
number=7, type=8, cpp_type=7, label=2,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='fee', full_name='Metadata.fee', index=7,
|
||||
name='fee', full_name='pb.Metadata.fee', index=7,
|
||||
number=8, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='thumbnail', full_name='Metadata.thumbnail', index=8,
|
||||
name='thumbnail', full_name='pb.Metadata.thumbnail', index=8,
|
||||
number=9, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='preview', full_name='Metadata.preview', index=9,
|
||||
name='preview', full_name='pb.Metadata.preview', index=9,
|
||||
number=10, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='licenseUrl', full_name='Metadata.licenseUrl', index=10,
|
||||
name='licenseUrl', full_name='pb.Metadata.licenseUrl', index=10,
|
||||
number=11, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='releaseTime', full_name='pb.Metadata.releaseTime', index=11,
|
||||
number=12, type=18, cpp_type=2, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
@@ -913,13 +920,13 @@ _METADATA = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=72,
|
||||
serialized_end=1958,
|
||||
serialized_start=34,
|
||||
serialized_end=1950,
|
||||
)
|
||||
|
||||
_METADATA.fields_by_name['version'].enum_type = _METADATA_VERSION
|
||||
_METADATA.fields_by_name['language'].enum_type = _METADATA_LANGUAGE
|
||||
_METADATA.fields_by_name['fee'].message_type = lbrynet_dot_schema_dot_proto_dot_fee__pb2._FEE
|
||||
_METADATA.fields_by_name['fee'].message_type = fee__pb2._FEE
|
||||
_METADATA_VERSION.containing_type = _METADATA
|
||||
_METADATA_LANGUAGE.containing_type = _METADATA
|
||||
DESCRIPTOR.message_types_by_name['Metadata'] = _METADATA
|
||||
@@ -927,8 +934,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Metadata = _reflection.GeneratedProtocolMessageType('Metadata', (_message.Message,), dict(
|
||||
DESCRIPTOR = _METADATA,
|
||||
__module__ = 'lbrynet.schema.proto.metadata_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Metadata)
|
||||
__module__ = 'metadata_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Metadata)
|
||||
))
|
||||
_sym_db.RegisterMessage(Metadata)
|
||||
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/certificate.proto";
|
||||
|
||||
message Signature {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required KeyType signatureType = 2;
|
||||
required bytes signature = 3;
|
||||
required bytes certificateId = 4;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/signature.proto
|
||||
# source: signature.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
|
||||
from . import certificate_pb2 as certificate__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/signature.proto',
|
||||
package='',
|
||||
name='signature.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n$lbrynet/schema/proto/signature.proto\x1a&lbrynet/schema/proto/certificate.proto\"\xa7\x01\n\tSignature\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.Signature.Version\x12\x1f\n\rsignatureType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
serialized_pb=_b('\n\x0fsignature.proto\x12\x02pb\x1a\x11\x63\x65rtificate.proto\"\xad\x01\n\tSignature\x12&\n\x07version\x18\x01 \x02(\x0e\x32\x15.pb.Signature.Version\x12\"\n\rsignatureType\x18\x02 \x02(\x0e\x32\x0b.pb.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,])
|
||||
dependencies=[certificate__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_SIGNATURE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Signature.Version',
|
||||
full_name='pb.Signature.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -43,42 +43,42 @@ _SIGNATURE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=206,
|
||||
serialized_end=248,
|
||||
serialized_start=174,
|
||||
serialized_end=216,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SIGNATURE_VERSION)
|
||||
|
||||
|
||||
_SIGNATURE = _descriptor.Descriptor(
|
||||
name='Signature',
|
||||
full_name='Signature',
|
||||
full_name='pb.Signature',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Signature.version', index=0,
|
||||
name='version', full_name='pb.Signature.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signatureType', full_name='Signature.signatureType', index=1,
|
||||
name='signatureType', full_name='pb.Signature.signatureType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signature', full_name='Signature.signature', index=2,
|
||||
name='signature', full_name='pb.Signature.signature', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='certificateId', full_name='Signature.certificateId', index=3,
|
||||
name='certificateId', full_name='pb.Signature.certificateId', index=3,
|
||||
number=4, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -97,20 +97,20 @@ _SIGNATURE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=81,
|
||||
serialized_end=248,
|
||||
serialized_start=43,
|
||||
serialized_end=216,
|
||||
)
|
||||
|
||||
_SIGNATURE.fields_by_name['version'].enum_type = _SIGNATURE_VERSION
|
||||
_SIGNATURE.fields_by_name['signatureType'].enum_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._KEYTYPE
|
||||
_SIGNATURE.fields_by_name['signatureType'].enum_type = certificate__pb2._KEYTYPE
|
||||
_SIGNATURE_VERSION.containing_type = _SIGNATURE
|
||||
DESCRIPTOR.message_types_by_name['Signature'] = _SIGNATURE
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Signature = _reflection.GeneratedProtocolMessageType('Signature', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SIGNATURE,
|
||||
__module__ = 'lbrynet.schema.proto.signature_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Signature)
|
||||
__module__ = 'signature_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Signature)
|
||||
))
|
||||
_sym_db.RegisterMessage(Signature)
|
||||
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
message Source {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
enum SourceTypes {
|
||||
UNKNOWN_SOURCE_TYPE = 0;
|
||||
lbry_sd_hash = 1;
|
||||
}
|
||||
required SourceTypes sourceType = 2;
|
||||
required bytes source = 3;
|
||||
required string contentType = 4;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/source.proto
|
||||
# source: source.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/source.proto',
|
||||
package='',
|
||||
name='source.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n!lbrynet/schema/proto/source.proto\"\xde\x01\n\x06Source\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Source.Version\x12\'\n\nsourceType\x18\x02 \x02(\x0e\x32\x13.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
|
||||
serialized_pb=_b('\n\x0csource.proto\x12\x02pb\"\xe4\x01\n\x06Source\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.pb.Source.Version\x12*\n\nsourceType\x18\x02 \x02(\x0e\x32\x16.pb.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
|
||||
)
|
||||
|
||||
|
||||
|
||||
_SOURCE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Source.Version',
|
||||
full_name='pb.Source.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -41,14 +41,14 @@ _SOURCE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=160,
|
||||
serialized_end=202,
|
||||
serialized_start=149,
|
||||
serialized_end=191,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SOURCE_VERSION)
|
||||
|
||||
_SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
|
||||
name='SourceTypes',
|
||||
full_name='Source.SourceTypes',
|
||||
full_name='pb.Source.SourceTypes',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -63,42 +63,42 @@ _SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=204,
|
||||
serialized_end=260,
|
||||
serialized_start=193,
|
||||
serialized_end=249,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SOURCE_SOURCETYPES)
|
||||
|
||||
|
||||
_SOURCE = _descriptor.Descriptor(
|
||||
name='Source',
|
||||
full_name='Source',
|
||||
full_name='pb.Source',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Source.version', index=0,
|
||||
name='version', full_name='pb.Source.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='sourceType', full_name='Source.sourceType', index=1,
|
||||
name='sourceType', full_name='pb.Source.sourceType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='source', full_name='Source.source', index=2,
|
||||
name='source', full_name='pb.Source.source', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='contentType', full_name='Source.contentType', index=3,
|
||||
name='contentType', full_name='pb.Source.contentType', index=3,
|
||||
number=4, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -118,8 +118,8 @@ _SOURCE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=38,
|
||||
serialized_end=260,
|
||||
serialized_start=21,
|
||||
serialized_end=249,
|
||||
)
|
||||
|
||||
_SOURCE.fields_by_name['version'].enum_type = _SOURCE_VERSION
|
||||
@@ -131,8 +131,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SOURCE,
|
||||
__module__ = 'lbrynet.schema.proto.source_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Source)
|
||||
__module__ = 'source_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Source)
|
||||
))
|
||||
_sym_db.RegisterMessage(Source)
|
||||
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/metadata.proto";
|
||||
import "lbrynet/schema/proto/source.proto";
|
||||
|
||||
|
||||
message Stream {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Metadata metadata = 2;
|
||||
required Source source = 3;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/stream.proto
|
||||
# source: stream.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,24 +12,24 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import metadata_pb2 as lbrynet_dot_schema_dot_proto_dot_metadata__pb2
|
||||
from lbrynet.schema.proto import source_pb2 as lbrynet_dot_schema_dot_proto_dot_source__pb2
|
||||
from . import metadata_pb2 as metadata__pb2
|
||||
from . import source_pb2 as source__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/stream.proto',
|
||||
package='',
|
||||
name='stream.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n!lbrynet/schema/proto/stream.proto\x1a#lbrynet/schema/proto/metadata.proto\x1a!lbrynet/schema/proto/source.proto\"\x8c\x01\n\x06Stream\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Stream.Version\x12\x1b\n\x08metadata\x18\x02 \x02(\x0b\x32\t.Metadata\x12\x17\n\x06source\x18\x03 \x02(\x0b\x32\x07.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
serialized_pb=_b('\n\x0cstream.proto\x12\x02pb\x1a\x0emetadata.proto\x1a\x0csource.proto\"\x95\x01\n\x06Stream\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.pb.Stream.Version\x12\x1e\n\x08metadata\x18\x02 \x02(\x0b\x32\x0c.pb.Metadata\x12\x1a\n\x06source\x18\x03 \x02(\x0b\x32\n.pb.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_metadata__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_source__pb2.DESCRIPTOR,])
|
||||
dependencies=[metadata__pb2.DESCRIPTOR,source__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_STREAM_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Stream.Version',
|
||||
full_name='pb.Stream.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -44,35 +44,35 @@ _STREAM_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=208,
|
||||
serialized_end=250,
|
||||
serialized_start=158,
|
||||
serialized_end=200,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_STREAM_VERSION)
|
||||
|
||||
|
||||
_STREAM = _descriptor.Descriptor(
|
||||
name='Stream',
|
||||
full_name='Stream',
|
||||
full_name='pb.Stream',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Stream.version', index=0,
|
||||
name='version', full_name='pb.Stream.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='metadata', full_name='Stream.metadata', index=1,
|
||||
name='metadata', full_name='pb.Stream.metadata', index=1,
|
||||
number=2, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='source', full_name='Stream.source', index=2,
|
||||
name='source', full_name='pb.Stream.source', index=2,
|
||||
number=3, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -91,21 +91,21 @@ _STREAM = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=110,
|
||||
serialized_end=250,
|
||||
serialized_start=51,
|
||||
serialized_end=200,
|
||||
)
|
||||
|
||||
_STREAM.fields_by_name['version'].enum_type = _STREAM_VERSION
|
||||
_STREAM.fields_by_name['metadata'].message_type = lbrynet_dot_schema_dot_proto_dot_metadata__pb2._METADATA
|
||||
_STREAM.fields_by_name['source'].message_type = lbrynet_dot_schema_dot_proto_dot_source__pb2._SOURCE
|
||||
_STREAM.fields_by_name['metadata'].message_type = metadata__pb2._METADATA
|
||||
_STREAM.fields_by_name['source'].message_type = source__pb2._SOURCE
|
||||
_STREAM_VERSION.containing_type = _STREAM
|
||||
DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
|
||||
DESCRIPTOR = _STREAM,
|
||||
__module__ = 'lbrynet.schema.proto.stream_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Stream)
|
||||
__module__ = 'stream_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Stream)
|
||||
))
|
||||
_sym_db.RegisterMessage(Stream)
|
||||
|
||||
|
||||
@@ -14,4 +14,8 @@ class Metadata(Schema):
|
||||
if 'fee' in _metadata:
|
||||
fee_pb = Fee.load(_metadata.pop('fee'))
|
||||
_message_pb.fee.CopyFrom(fee_pb)
|
||||
return cls._load(_metadata, _message_pb)
|
||||
_message_pb.releaseTime = int(_metadata.get('releaseTime', 0))
|
||||
built_message = cls._load(_metadata, _message_pb)
|
||||
if built_message.releaseTime == 0:
|
||||
built_message.ClearField('releaseTime')
|
||||
return built_message
|
||||
@@ -0,0 +1,44 @@
|
||||
from collections import namedtuple
|
||||
|
||||
from typing import Union
|
||||
|
||||
LEGACY = namedtuple('Legacy', 'payload')
|
||||
NAMED_SECP256K1 = namedtuple('NamedSECP256k1', 'raw_signature certificate_id payload')
|
||||
FLAGS = {
|
||||
LEGACY: 0x80,
|
||||
NAMED_SECP256K1: 0x01
|
||||
}
|
||||
|
||||
class Signature:
|
||||
|
||||
def __init__(self, data: Union[LEGACY, NAMED_SECP256K1]):
|
||||
self.data = data
|
||||
|
||||
@property
|
||||
def payload(self):
|
||||
return self.data.payload
|
||||
|
||||
@property
|
||||
def certificate_id(self):
|
||||
if type(self.data) == NAMED_SECP256K1:
|
||||
return self.data.certificate_id
|
||||
|
||||
@property
|
||||
def raw_signature(self):
|
||||
if type(self.data) == NAMED_SECP256K1:
|
||||
return self.data.raw_signature
|
||||
|
||||
@classmethod
|
||||
def flagged_parse(cls, binary: bytes):
|
||||
flag = binary[0]
|
||||
if flag == FLAGS[NAMED_SECP256K1]:
|
||||
return cls(NAMED_SECP256K1(binary[1:65], binary[65:85], binary[85:]))
|
||||
else:
|
||||
return cls(LEGACY(binary))
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
if isinstance(self.data, NAMED_SECP256K1):
|
||||
return (bytes([FLAGS[type(self.data)]]) + self.data.raw_signature + self.data.certificate_id + self.payload)
|
||||
elif isinstance(self.data, LEGACY):
|
||||
return self.payload
|
||||
+29
-10
@@ -5,6 +5,7 @@ from lbrynet.schema.address import decode_address
|
||||
from lbrynet.schema.encoding import decode_b64_fields
|
||||
from lbrynet.schema.schema.certificate import Certificate
|
||||
from lbrynet.schema.schema.claim import Claim
|
||||
from lbrynet.schema.signature import Signature, NAMED_SECP256K1
|
||||
from lbrynet.schema.validator import validate_claim_id
|
||||
from lbrynet.schema.schema import V_0_0_1, CLAIM_TYPE, CLAIM_TYPES, CERTIFICATE_TYPE, VERSION
|
||||
from lbrynet.schema.schema import NIST256p, NIST384p, SECP256k1, SHA256, SHA384
|
||||
@@ -45,24 +46,41 @@ class NIST_ECDSASigner(object):
|
||||
def generate(cls):
|
||||
return cls(ecdsa.SigningKey.generate(curve=cls.CURVE, hashfunc=cls.HASHFUNC_NAME))
|
||||
|
||||
def sign_stream_claim(self, claim, claim_address, cert_claim_id):
|
||||
def sign(self, *fields):
|
||||
digest = self.HASHFUNC(bytearray(b''.join(fields))).digest()
|
||||
return self.private_key.sign_digest_deterministic(digest, hashfunc=self.HASHFUNC)
|
||||
|
||||
def sign_stream_claim(self, claim, claim_address, cert_claim_id, name, legacy=False):
|
||||
validate_claim_id(cert_claim_id)
|
||||
raw_cert_id = binascii.unhexlify(cert_claim_id)
|
||||
decoded_addr = decode_address(claim_address)
|
||||
if not legacy:
|
||||
assert name, "Name is required for detached signatures"
|
||||
assert self.CURVE_NAME == SECP256k1, f"Only SECP256k1 is supported, not: {self.CURVE_NAME}"
|
||||
signature = self.sign(
|
||||
name.lower().encode(),
|
||||
decoded_addr,
|
||||
claim.serialized_no_signature,
|
||||
raw_cert_id,
|
||||
)
|
||||
else:
|
||||
signature = self.sign(decoded_addr, claim.serialized_no_signature, raw_cert_id)
|
||||
|
||||
to_sign = bytearray()
|
||||
to_sign.extend(decoded_addr)
|
||||
to_sign.extend(claim.serialized_no_signature)
|
||||
to_sign.extend(binascii.unhexlify(cert_claim_id))
|
||||
|
||||
digest = self.HASHFUNC(to_sign).digest()
|
||||
if not legacy:
|
||||
return Claim.load(decode_b64_fields(claim.protobuf_dict)), Signature(NAMED_SECP256K1(
|
||||
signature,
|
||||
raw_cert_id,
|
||||
claim.serialized_no_signature
|
||||
))
|
||||
# -- Legacy signer (signature inside protobuf) --
|
||||
|
||||
if not isinstance(self.private_key, ecdsa.SigningKey):
|
||||
raise Exception("Not given a signing key")
|
||||
sig_dict = {
|
||||
"version": V_0_0_1,
|
||||
"signatureType": self.CURVE_NAME,
|
||||
"signature": self.private_key.sign_digest_deterministic(digest, hashfunc=self.HASHFUNC),
|
||||
"certificateId": binascii.unhexlify(cert_claim_id)
|
||||
"signature": signature,
|
||||
"certificateId": raw_cert_id
|
||||
}
|
||||
|
||||
msg = {
|
||||
@@ -71,7 +89,8 @@ class NIST_ECDSASigner(object):
|
||||
"publisherSignature": sig_dict
|
||||
}
|
||||
|
||||
return Claim.load(msg)
|
||||
proto = Claim.load(msg)
|
||||
return proto, Signature.flagged_parse(proto.SerializeToString())
|
||||
|
||||
|
||||
class NIST256pSigner(NIST_ECDSASigner):
|
||||
|
||||
+13
-2
@@ -28,7 +28,7 @@ class URI(object):
|
||||
self.claim_id = claim_id
|
||||
self.path = path
|
||||
|
||||
if self.path is not None and not self.is_channel:
|
||||
if self.path is not None and not self.contains_channel:
|
||||
raise ValueError("Content claims cannot have paths")
|
||||
|
||||
def __str__(self):
|
||||
@@ -39,10 +39,21 @@ class URI(object):
|
||||
if not hasattr(other, prop) or getattr(self, prop) != getattr(other, prop):
|
||||
return False
|
||||
return self.__class__ == other.__class__
|
||||
@property
|
||||
def channel_name(self):
|
||||
return self.name if self.contains_channel else None
|
||||
|
||||
@property
|
||||
def claim_name(self):
|
||||
return self.name if not self.contains_channel else self.path
|
||||
|
||||
@property
|
||||
def contains_channel(self):
|
||||
return self.name.startswith(CHANNEL_CHAR)
|
||||
|
||||
@property
|
||||
def is_channel(self):
|
||||
return self.name.startswith(CHANNEL_CHAR)
|
||||
return self.contains_channel and not self.path
|
||||
|
||||
def to_uri_string(self):
|
||||
uri_string = PROTOCOL + "%s" % self.name
|
||||
|
||||
@@ -76,18 +76,25 @@ class Validator:
|
||||
from ecdsa import BadSignatureError
|
||||
raise BadSignatureError
|
||||
|
||||
def validate_claim_signature(self, claim, claim_address):
|
||||
def validate_claim_signature(self, claim, claim_address, name):
|
||||
to_sign = bytearray()
|
||||
if claim.detached_signature and claim.detached_signature.raw_signature:
|
||||
assert name is not None, "Name is required for verifying detached signatures."
|
||||
to_sign.extend(name.lower().encode())
|
||||
signature = claim.detached_signature.raw_signature
|
||||
payload = claim.detached_signature.payload
|
||||
else:
|
||||
# extract and serialize the stream from the claim, then check the signature
|
||||
signature = binascii.unhexlify(claim.signature)
|
||||
payload = claim.serialized_no_signature
|
||||
decoded_address = decode_address(claim_address)
|
||||
|
||||
# extract and serialize the stream from the claim, then check the signature
|
||||
signature = binascii.unhexlify(claim.signature)
|
||||
|
||||
if signature is None:
|
||||
raise Exception("No signature to validate")
|
||||
|
||||
to_sign = bytearray()
|
||||
to_sign.extend(decoded_address)
|
||||
to_sign.extend(claim.serialized_no_signature)
|
||||
to_sign.extend(payload)
|
||||
to_sign.extend(binascii.unhexlify(self.certificate_claim_id))
|
||||
|
||||
return self.validate_signature(self.HASHFUNC(to_sign).digest(), signature)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
FROM debian:buster-slim
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get upgrade -y && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
git \
|
||||
python3.7 \
|
||||
python3.7-dev \
|
||||
python3-pip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN python3.7 -m pip install --upgrade pip setuptools wheel
|
||||
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
|
||||
RUN python3.7 -m pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
RUN python3.7 -m pip install -e .
|
||||
|
||||
# Orchstr8 API
|
||||
EXPOSE 7954
|
||||
# Wallet Server
|
||||
EXPOSE 5280
|
||||
# SPV Server
|
||||
EXPOSE 50002
|
||||
# blockchain
|
||||
EXPOSE 9246
|
||||
ENV TORBA_LEDGER lbrynet.extras.wallet
|
||||
|
||||
RUN /usr/local/bin/orchstr8 download
|
||||
ENTRYPOINT ["/usr/local/bin/orchstr8"]
|
||||
@@ -0,0 +1,49 @@
|
||||
FROM debian:buster-slim
|
||||
|
||||
ARG user=lbry
|
||||
|
||||
# create an unprivileged user
|
||||
RUN groupadd -g 999 $user && useradd -r -u 999 -g $user $user
|
||||
RUN mkdir -p /home/$user/
|
||||
RUN chown -R $user:$user /home/$user/
|
||||
|
||||
# install python, pip, git and clean up
|
||||
RUN apt-get update && apt-get -y --no-install-recommends install build-essential git python3.7 python3.7-dev python3-pip && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# create and chown database dir
|
||||
ARG db_dir=/database
|
||||
RUN mkdir -p $db_dir
|
||||
RUN chown -R $user:$user $db_dir
|
||||
|
||||
# change into our user
|
||||
USER $user
|
||||
WORKDIR /home/$user
|
||||
|
||||
# RUN pip3 install -U pip (broken on pip 10 https://github.com/pypa/pip/issues/5240)
|
||||
RUN python3.7 -m pip install --upgrade pip setuptools
|
||||
|
||||
# get torba
|
||||
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git#egg=torba
|
||||
|
||||
# copy lbrynet
|
||||
RUN mkdir projects/
|
||||
COPY . projects/lbry
|
||||
USER root
|
||||
RUN chown -R $user:$user .
|
||||
USER $user
|
||||
WORKDIR projects/lbry
|
||||
RUN python3.7 -m pip install --user -e . && rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
ARG host=0.0.0.0
|
||||
ARG tcp_port=50001
|
||||
ARG daemon_url=http://lbry:lbry@localhost:9245/
|
||||
VOLUME $db_dir
|
||||
ENV TCP_PORT=$tcp_port
|
||||
ENV HOST=$host
|
||||
ENV DAEMON_URL=$daemon_url
|
||||
ENV DB_DIRECTORY=$db_dir
|
||||
ENV BANDWIDTH_LIMIT=1000000000000000000000000000000000000000000
|
||||
ENV MAX_SESSIONS=1000000000
|
||||
ENV MAX_SEND=1000000000000000000
|
||||
ENTRYPOINT ["/home/lbry/.local/bin/torba-server"]
|
||||
@@ -1,57 +0,0 @@
|
||||
"""Decrypt a single blob"""
|
||||
import argparse
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import reactor
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.cryptstream import CryptBlob
|
||||
from lbrynet.blob import BlobFile
|
||||
from lbrynet.core import log_support
|
||||
|
||||
|
||||
log = logging.getLogger('decrypt_blob')
|
||||
|
||||
|
||||
def main():
|
||||
conf.initialize_settings()
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('blob_file')
|
||||
parser.add_argument('hex_key')
|
||||
parser.add_argument('hex_iv')
|
||||
parser.add_argument('output')
|
||||
args = parser.parse_args()
|
||||
log_support.configure_console()
|
||||
|
||||
d = run(args)
|
||||
reactor.run()
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run(args):
|
||||
try:
|
||||
yield decrypt_blob(args.blob_file, args.hex_key, args.hex_iv, args.output)
|
||||
except Exception:
|
||||
log.exception('Failed to decrypt blob')
|
||||
finally:
|
||||
reactor.callLater(0, reactor.stop)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def decrypt_blob(blob_file, key, iv, output):
|
||||
filename = os.path.abspath(blob_file)
|
||||
length = os.path.getsize(filename)
|
||||
directory, blob_hash = os.path.split(filename)
|
||||
blob = BlobFile(directory, blob_hash, length)
|
||||
decryptor = CryptBlob.StreamBlobDecryptor(
|
||||
blob, binascii.unhexlify(key), binascii.unhexlify(iv), length)
|
||||
with open(output, 'w') as f:
|
||||
yield decryptor.decrypt(f.write)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,85 +0,0 @@
|
||||
import curses
|
||||
import time
|
||||
import datetime
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
stdscr = curses.initscr()
|
||||
|
||||
api = JSONRPCProxy.from_url("http://localhost:5280")
|
||||
|
||||
|
||||
def init_curses():
|
||||
curses.noecho()
|
||||
curses.cbreak()
|
||||
stdscr.nodelay(1)
|
||||
stdscr.keypad(1)
|
||||
|
||||
|
||||
def teardown_curses():
|
||||
curses.nocbreak()
|
||||
stdscr.keypad(0)
|
||||
curses.echo()
|
||||
curses.endwin()
|
||||
|
||||
|
||||
def refresh(node_index):
|
||||
height, width = stdscr.getmaxyx()
|
||||
node_ids = api.get_node_ids()
|
||||
node_id = node_ids[node_index]
|
||||
node_statuses = api.node_status()
|
||||
running = node_statuses[node_id]
|
||||
buckets = api.node_routing_table(node_id=node_id)
|
||||
|
||||
for y in range(height):
|
||||
stdscr.addstr(y, 0, " " * (width - 1))
|
||||
|
||||
stdscr.addstr(0, 0, "node id: %s, running: %s (%i/%i running)" % (node_id, running, sum(node_statuses.values()), len(node_ids)))
|
||||
stdscr.addstr(1, 0, "%i buckets, %i contacts" %
|
||||
(len(buckets), sum([len(buckets[b]['contacts']) for b in buckets])))
|
||||
|
||||
y = 3
|
||||
for i in sorted(buckets.keys()):
|
||||
stdscr.addstr(y, 0, "bucket %s" % i)
|
||||
y += 1
|
||||
for h in sorted(buckets[i]['contacts'], key=lambda x: x['node_id'].decode('hex')):
|
||||
stdscr.addstr(y, 0, '%s (%s:%i) failures: %i, last replied to us: %s, last requested from us: %s' %
|
||||
(h['node_id'], h['address'], h['port'], h['failedRPCs'],
|
||||
datetime.datetime.fromtimestamp(float(h['lastReplied'] or 0)),
|
||||
datetime.datetime.fromtimestamp(float(h['lastRequested'] or 0))))
|
||||
y += 1
|
||||
y += 1
|
||||
|
||||
stdscr.addstr(y + 1, 0, str(time.time()))
|
||||
stdscr.refresh()
|
||||
return len(node_ids)
|
||||
|
||||
|
||||
def do_main():
|
||||
c = None
|
||||
nodes = 1
|
||||
node_index = 0
|
||||
while c not in [ord('q'), ord('Q')]:
|
||||
try:
|
||||
nodes = refresh(node_index)
|
||||
except:
|
||||
pass
|
||||
c = stdscr.getch()
|
||||
if c == curses.KEY_LEFT:
|
||||
node_index -= 1
|
||||
node_index = max(node_index, 0)
|
||||
elif c == curses.KEY_RIGHT:
|
||||
node_index += 1
|
||||
node_index = min(node_index, nodes - 1)
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
init_curses()
|
||||
do_main()
|
||||
finally:
|
||||
teardown_curses()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loaded 100 of 137 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user