Compare commits
123
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
574c2ce794 | ||
|
|
7f5541eb5c | ||
|
|
7b49241c38 | ||
|
|
da716b2e3d | ||
|
|
a144fae8b2 | ||
|
|
0d90b82287 | ||
|
|
567c23e10f | ||
|
|
16970b7d37 | ||
|
|
8593cd362a | ||
|
|
2dbf96da81 | ||
|
|
83c7797187 | ||
|
|
6606deafa5 | ||
|
|
19b0d59159 | ||
|
|
9ccb3fa2a3 | ||
|
|
48533df523 | ||
|
|
1ae1137005 | ||
|
|
d5f2832b2f | ||
|
|
d0230b4893 | ||
|
|
a6cd53b97c | ||
|
|
7a28171a72 | ||
|
|
150bcb1116 | ||
|
|
6fe9c5bf00 | ||
|
|
3a86fc6680 | ||
|
|
2805cbb331 | ||
|
|
68d7a7014c | ||
|
|
883fa267a5 | ||
|
|
5f2c3ac965 | ||
|
|
bf8e32ebd2 | ||
|
|
5971be5bef | ||
|
|
677d76a3e4 | ||
|
|
6ed3207cb7 | ||
|
|
be6448b74f | ||
|
|
b707ee4844 | ||
|
|
0e9888be3b | ||
|
|
68b56b7858 | ||
|
|
d32697f611 | ||
|
|
3c54b2bcd7 | ||
|
|
899b182811 | ||
|
|
0dfb5e008d | ||
|
|
abf94357eb | ||
|
|
36aded3830 | ||
|
|
d7ebf50602 | ||
|
|
523c4c86a9 | ||
|
|
74cccdbfc7 | ||
|
|
70471eebfa | ||
|
|
6339224980 | ||
|
|
fe97f7a119 | ||
|
|
f8ed605da2 | ||
|
|
e38ef2af1f | ||
|
|
5d8e627d02 | ||
|
|
c9d4ad5450 | ||
|
|
31586bf588 | ||
|
|
974d3c83e3 | ||
|
|
8dc8ca382b | ||
|
|
1dbb250413 | ||
|
|
9f9ec692c7 | ||
|
|
2c1b54c76c | ||
|
|
7314494fb9 | ||
|
|
e74a5e8ef3 | ||
|
|
b9af999739 | ||
|
|
2de155f90f | ||
|
|
ab3db2e626 | ||
|
|
f9d26b04d8 | ||
|
|
24e0673198 | ||
|
|
833261801f | ||
|
|
6c8cbc1b9e | ||
|
|
6ae4e68d1c | ||
|
|
d22a2dee33 | ||
|
|
0f1ed589a5 | ||
|
|
b032e99dd1 | ||
|
|
fd7811f38d | ||
|
|
071db3a73d | ||
|
|
950c122850 | ||
|
|
e0386ea9dc | ||
|
|
6c3eee0f0e | ||
|
|
4c23add193 | ||
|
|
248d1a18c7 | ||
|
|
17d1b85423 | ||
|
|
b0deb63129 | ||
|
|
ebdb33bd11 | ||
|
|
69ccbda168 | ||
|
|
a38643cca6 | ||
|
|
712b19605e | ||
|
|
8a5f3842fc | ||
|
|
db3e578bcd | ||
|
|
7e22adff7c | ||
|
|
76fbaac658 | ||
|
|
c4fac29ecb | ||
|
|
33392b3351 | ||
|
|
0613cf2d59 | ||
|
|
a930ca0631 | ||
|
|
63a7271008 | ||
|
|
5ca0ebcbb2 | ||
|
|
2c84b7a7dd | ||
|
|
71a0c704bd | ||
|
|
d30e7cd29e | ||
|
|
aeb7c6fe76 | ||
|
|
134c20da93 | ||
|
|
893b4fc3ca | ||
|
|
9b27e81940 | ||
|
|
72c0acf269 | ||
|
|
7be53ad692 | ||
|
|
16e596ec87 | ||
|
|
3fdcb80960 | ||
|
|
26a3221bac | ||
|
|
eb16cb0aa8 | ||
|
|
b5340505e5 | ||
|
|
4bb8129bb7 | ||
|
|
7f838d3a31 | ||
|
|
c186c31bf7 | ||
|
|
86fb99a37c | ||
|
|
6c52450858 | ||
|
|
fb4e94b04a | ||
|
|
248baf58b4 | ||
|
|
0845d65f4e | ||
|
|
dcc5415648 | ||
|
|
3c38a88b22 | ||
|
|
187b7670b1 | ||
|
|
049c17ff08 | ||
|
|
8de97bb441 | ||
|
|
97c0092923 | ||
|
|
e65e1fb5a2 | ||
|
|
20dd1e55f5 |
No files matched your search
@@ -0,0 +1,7 @@
|
||||
.git
|
||||
.tox
|
||||
__pycache__
|
||||
dist
|
||||
lbrynet.egg-info
|
||||
docs
|
||||
tests
|
||||
+7
-5
@@ -9,8 +9,8 @@ jobs:
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid==2.0.4
|
||||
# newer astroid fails in pylint so we pre-install older version
|
||||
- pip install astroid==2.0.4 aiohttp==3.4.4
|
||||
# newer astroid and aiohttp fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
@@ -24,8 +24,9 @@ jobs:
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .[test]
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -v tests.unit.wallet
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics tests.unit.components tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.test_cli tests.unit.test_customLogger tests.unit.schema
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.schema tests.unit.wallet tests.unit.components tests.unit.test_conf
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.test_cli
|
||||
#- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
@@ -110,7 +111,8 @@ jobs:
|
||||
env: OS=mac
|
||||
cache: false
|
||||
before_install:
|
||||
- brew upgrade python
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
|
||||
@@ -17,9 +17,6 @@ dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
|
||||
use_auth_http: True
|
||||
use_https: True
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
|
||||
+1
-7
@@ -1,10 +1,4 @@
|
||||
import logging
|
||||
from lbrynet.custom_logger import install_logger
|
||||
|
||||
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.30.3"
|
||||
__version__ = "0.30.5"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
install_logger()
|
||||
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -9,6 +9,7 @@ from binascii import hexlify
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
|
||||
from lbrynet.blob.CryptStreamCreator import CryptStreamCreator
|
||||
@@ -119,13 +120,13 @@ def create_lbry_file(blob_manager, storage, payment_rate_manager, lbry_file_mana
|
||||
sd_hash = yield descriptor_writer.create_descriptor(sd_info)
|
||||
|
||||
log.debug("saving the stream")
|
||||
yield storage.store_stream(
|
||||
yield f2d(storage.store_stream(
|
||||
sd_info['stream_hash'], sd_hash, sd_info['stream_name'], sd_info['key'],
|
||||
sd_info['suggested_file_name'], sd_info['blobs']
|
||||
)
|
||||
))
|
||||
log.debug("adding to the file manager")
|
||||
lbry_file = yield lbry_file_manager.add_published_file(
|
||||
lbry_file = yield f2d(lbry_file_manager.add_published_file(
|
||||
sd_info['stream_hash'], sd_hash, hexlify(file_directory.encode()), payment_rate_manager,
|
||||
payment_rate_manager.min_blob_data_payment_rate
|
||||
)
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
@@ -5,12 +5,12 @@ import logging
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from lbrynet import conf
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
|
||||
from lbrynet.utils import short_hash
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
|
||||
@@ -34,12 +34,12 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
STATUS_STOPPED = "stopped"
|
||||
STATUS_FINISHED = "finished"
|
||||
|
||||
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage, lbry_file_manager,
|
||||
payment_rate_manager, wallet, download_directory, file_name, stream_name, sd_hash, key,
|
||||
suggested_file_name, download_mirrors=None):
|
||||
def __init__(self, conf: Config, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
|
||||
lbry_file_manager, payment_rate_manager, wallet, download_directory, file_name, stream_name,
|
||||
sd_hash, key, suggested_file_name, download_mirrors=None):
|
||||
super().__init__(
|
||||
stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name
|
||||
conf, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager,
|
||||
wallet, download_directory, key, stream_name, file_name
|
||||
)
|
||||
self.sd_hash = sd_hash
|
||||
self.rowid = rowid
|
||||
@@ -55,9 +55,9 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.channel_name = None
|
||||
self.metadata = None
|
||||
self.mirror = None
|
||||
if download_mirrors or conf.settings['download_mirrors']:
|
||||
if download_mirrors or conf.download_mirrors:
|
||||
self.mirror = HTTPBlobDownloader(
|
||||
self.blob_manager, servers=download_mirrors or conf.settings['download_mirrors']
|
||||
self.blob_manager, servers=download_mirrors or conf.download_mirrors
|
||||
)
|
||||
|
||||
def set_claim_info(self, claim_info):
|
||||
@@ -70,13 +70,11 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.channel_name = claim_info['channel_name']
|
||||
self.metadata = claim_info['value']['stream']['metadata']
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claim_info(self, include_supports=True):
|
||||
claim_info = yield self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
async def get_claim_info(self, include_supports=True):
|
||||
claim_info = await self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
if claim_info:
|
||||
self.set_claim_info(claim_info)
|
||||
|
||||
defer.returnValue(claim_info)
|
||||
return claim_info
|
||||
|
||||
@property
|
||||
def saving_status(self):
|
||||
@@ -101,16 +99,15 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
if self.mirror:
|
||||
self.mirror.stop()
|
||||
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
|
||||
yield EncryptedFileDownloader.stop(self, err=err)
|
||||
yield super().stop(err)
|
||||
if change_status is True:
|
||||
status = yield self._save_status()
|
||||
defer.returnValue(status)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def status(self):
|
||||
blobs = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
async def status(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
|
||||
completed_blobs = yield self.blob_manager.completed_blobs(blob_hashes)
|
||||
completed_blobs = self.blob_manager.completed_blobs(blob_hashes)
|
||||
num_blobs_completed = len(completed_blobs)
|
||||
num_blobs_known = len(blob_hashes)
|
||||
|
||||
@@ -120,9 +117,10 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
status = "stopped"
|
||||
else:
|
||||
status = "running"
|
||||
defer.returnValue(EncryptedFileStatusReport(
|
||||
|
||||
return EncryptedFileStatusReport(
|
||||
self.file_name, num_blobs_completed, num_blobs_known, status
|
||||
))
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start(self):
|
||||
@@ -150,7 +148,7 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_RUNNING
|
||||
status = yield self.lbry_file_manager.change_lbry_file_status(self, status)
|
||||
self._saving_status = False
|
||||
defer.returnValue(status)
|
||||
return status
|
||||
|
||||
def save_status(self):
|
||||
return self._save_status()
|
||||
@@ -180,10 +178,10 @@ class ManagedEncryptedFileDownloaderFactory:
|
||||
if file_name:
|
||||
file_name = hexlify(file_name.encode())
|
||||
hex_download_directory = hexlify(download_directory.encode())
|
||||
lbry_file = yield self.lbry_file_manager.add_downloaded_file(
|
||||
lbry_file = yield f2d(self.lbry_file_manager.add_downloaded_file(
|
||||
stream_hash, metadata.source_blob_hash, hex_download_directory, payment_rate_manager,
|
||||
data_rate, file_name=file_name, download_mirrors=download_mirrors
|
||||
)
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -3,11 +3,13 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
import random
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet import conf
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.reflector.reupload import reflect_file
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
@@ -27,9 +29,11 @@ class EncryptedFileManager:
|
||||
# when reflecting files, reflect up to this many files at a time
|
||||
CONCURRENT_REFLECTS = 5
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, wallet, payment_rate_manager, storage, sd_identifier):
|
||||
self.auto_re_reflect = conf.settings['reflect_uploads'] and conf.settings['auto_re_reflect_interval'] > 0
|
||||
self.auto_re_reflect_interval = conf.settings['auto_re_reflect_interval']
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, wallet,
|
||||
payment_rate_manager, storage, sd_identifier):
|
||||
self.conf = conf
|
||||
self.auto_re_reflect = conf.reflect_uploads and conf.auto_re_reflect_interval > 0
|
||||
self.auto_re_reflect_interval = conf.auto_re_reflect_interval
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
@@ -41,11 +45,9 @@ class EncryptedFileManager:
|
||||
self.lbry_files = []
|
||||
self.lbry_file_reflector = task.LoopingCall(self.reflect_lbry_files)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
yield self._add_to_sd_identifier()
|
||||
yield self._start_lbry_files()
|
||||
log.info("Started file manager")
|
||||
self._add_to_sd_identifier()
|
||||
return self._start_lbry_files()
|
||||
|
||||
def get_lbry_file_status(self, lbry_file):
|
||||
return self.storage.get_lbry_file_status(lbry_file.rowid)
|
||||
@@ -55,7 +57,7 @@ class EncryptedFileManager:
|
||||
|
||||
def change_lbry_file_status(self, lbry_file, status):
|
||||
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
|
||||
return self.storage.change_file_status(lbry_file.rowid, status)
|
||||
return f2d(self.storage.change_file_status(lbry_file.rowid, status))
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
@@ -79,6 +81,7 @@ class EncryptedFileManager:
|
||||
def _get_lbry_file(self, rowid, stream_hash, payment_rate_manager, sd_hash, key,
|
||||
stream_name, file_name, download_directory, suggested_file_name, download_mirrors=None):
|
||||
return ManagedEncryptedFileDownloader(
|
||||
self.conf,
|
||||
rowid,
|
||||
stream_hash,
|
||||
self.peer_finder,
|
||||
@@ -115,10 +118,9 @@ class EncryptedFileManager:
|
||||
except Exception:
|
||||
log.warning("Failed to start %i", file_info.get('rowid'))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start_lbry_files(self):
|
||||
files = yield self.storage.get_all_lbry_files()
|
||||
claim_infos = yield self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
async def _start_lbry_files(self):
|
||||
files = await self.storage.get_all_lbry_files()
|
||||
claim_infos = await self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
prm = self.payment_rate_manager
|
||||
|
||||
log.info("Starting %i files", len(files))
|
||||
@@ -151,14 +153,13 @@ class EncryptedFileManager:
|
||||
log.info("Stopping %i lbry files", len(self.lbry_files))
|
||||
yield defer.DeferredList([self._stop_lbry_file(lbry_file) for lbry_file in list(self.lbry_files)])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
async def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = stream_metadata['suggested_file_name']
|
||||
rowid = yield self.storage.save_published_file(
|
||||
rowid = await self.storage.save_published_file(
|
||||
stream_hash, file_name, download_directory, blob_data_rate, status
|
||||
)
|
||||
lbry_file = self._get_lbry_file(
|
||||
@@ -166,37 +167,36 @@ class EncryptedFileManager:
|
||||
stream_metadata['suggested_file_name'], download_mirrors=None
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info()
|
||||
await lbry_file.get_claim_info()
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
async def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
blob_data_rate=None, status=None, file_name=None, download_mirrors=None):
|
||||
status = status or ManagedEncryptedFileDownloader.STATUS_STOPPED
|
||||
payment_rate_manager = payment_rate_manager or self.payment_rate_manager
|
||||
blob_data_rate = blob_data_rate or payment_rate_manager.min_blob_data_payment_rate
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = file_name or stream_metadata['suggested_file_name']
|
||||
|
||||
# when we save the file we'll atomic touch the nearest file to the suggested file name
|
||||
# that doesn't yet exist in the download directory
|
||||
rowid = yield self.storage.save_downloaded_file(
|
||||
rowid = await self.storage.save_downloaded_file(
|
||||
stream_hash, hexlify(os.path.basename(unhexlify(file_name))), download_directory, blob_data_rate
|
||||
)
|
||||
file_name = (yield self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
file_name = (await self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
lbry_file = self._get_lbry_file(
|
||||
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
|
||||
stream_metadata['suggested_file_name'], download_mirrors
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info(include_supports=False)
|
||||
await lbry_file.get_claim_info(include_supports=False)
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_lbry_file(self, lbry_file, delete_file=False):
|
||||
@@ -222,7 +222,7 @@ class EncryptedFileManager:
|
||||
del self.storage.content_claim_callbacks[lbry_file.stream_hash]
|
||||
|
||||
yield lbry_file.delete_data()
|
||||
yield self.storage.delete_stream(lbry_file.stream_hash)
|
||||
yield f2d(self.storage.delete_stream(lbry_file.stream_hash))
|
||||
|
||||
if delete_file and os.path.isfile(full_path):
|
||||
os.remove(full_path)
|
||||
@@ -240,10 +240,10 @@ class EncryptedFileManager:
|
||||
def reflect_lbry_files(self):
|
||||
sem = defer.DeferredSemaphore(self.CONCURRENT_REFLECTS)
|
||||
ds = []
|
||||
sd_hashes_to_reflect = yield self.storage.get_streams_to_re_reflect()
|
||||
sd_hashes_to_reflect = yield f2d(self.storage.get_streams_to_re_reflect())
|
||||
for lbry_file in self.lbry_files:
|
||||
if lbry_file.sd_hash in sd_hashes_to_reflect:
|
||||
ds.append(sem.run(reflect_file, lbry_file))
|
||||
ds.append(sem.run(reflect_file, lbry_file, random.choice(self.conf.reflector_servers)))
|
||||
yield defer.DeferredList(ds)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import logging
|
||||
from twisted.internet import defer, threads
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
@@ -111,23 +111,13 @@ class BlobFile:
|
||||
if not self.writers and not self.readers:
|
||||
self._verified = False
|
||||
self.saved_verified_blob = False
|
||||
|
||||
def delete_from_file_system():
|
||||
try:
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
log.warning("An error occurred deleting %s: %s",
|
||||
str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
except Exception as e:
|
||||
log.exception("An error occurred deleting %s:", str(self.file_path), exc_info=e)
|
||||
else:
|
||||
return defer.fail(Failure(
|
||||
ValueError("File is currently being read or written and cannot be deleted")))
|
||||
raise ValueError("File is currently being read or written and cannot be deleted")
|
||||
|
||||
@property
|
||||
def verified(self):
|
||||
|
||||
@@ -3,6 +3,7 @@ from binascii import unhexlify
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.p2p.client.BlobRequester import BlobRequester
|
||||
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.p2p.client.DownloadManager import DownloadManager
|
||||
@@ -37,7 +38,7 @@ class CryptStreamDownloader:
|
||||
|
||||
#implements(IStreamDownloader)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
|
||||
key, stream_name):
|
||||
"""Initialize a CryptStreamDownloader
|
||||
|
||||
@@ -55,7 +56,7 @@ class CryptStreamDownloader:
|
||||
@return:
|
||||
|
||||
"""
|
||||
|
||||
self.conf = conf
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
|
||||
@@ -4,6 +4,8 @@ import traceback
|
||||
from binascii import hexlify, unhexlify
|
||||
from twisted.internet import defer, threads
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
@@ -17,9 +19,9 @@ log = logging.getLogger(__name__)
|
||||
class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
"""Classes which inherit from this class download LBRY files"""
|
||||
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager,
|
||||
storage, payment_rate_manager, wallet, key, stream_name, file_name):
|
||||
super().__init__(peer_finder, rate_limiter, blob_manager,
|
||||
super().__init__(conf, peer_finder, rate_limiter, blob_manager,
|
||||
payment_rate_manager, wallet, key, stream_name)
|
||||
self.stream_hash = stream_hash
|
||||
self.storage = storage
|
||||
@@ -28,16 +30,15 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_data(self):
|
||||
crypt_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
crypt_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
blob_hashes = [b.blob_hash for b in crypt_infos if b.blob_hash]
|
||||
sd_hash = yield self.storage.get_sd_blob_hash_for_stream(self.stream_hash)
|
||||
sd_hash = yield f2d(self.storage.get_sd_blob_hash_for_stream(self.stream_hash))
|
||||
blob_hashes.append(sd_hash)
|
||||
yield self.blob_manager.delete_blobs(blob_hashes)
|
||||
|
||||
def stop(self, err=None):
|
||||
d = self._close_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
|
||||
return d
|
||||
self._close_output()
|
||||
return super().stop(err=err)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
@@ -65,14 +66,9 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
def _close_output(self):
|
||||
pass
|
||||
|
||||
def get_total_bytes(self):
|
||||
d = self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def calculate_size(blobs):
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
d.addCallback(calculate_size)
|
||||
return d
|
||||
async def get_total_bytes(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
def get_total_bytes_cached(self):
|
||||
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
|
||||
@@ -102,7 +98,8 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
class EncryptedFileDownloaderFactory:
|
||||
#implements(IStreamDownloaderFactory)
|
||||
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet):
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet):
|
||||
self.conf = conf
|
||||
self.peer_finder = peer_finder
|
||||
self.rate_limiter = rate_limiter
|
||||
self.blob_manager = blob_manager
|
||||
@@ -135,9 +132,9 @@ class EncryptedFileDownloaderFactory:
|
||||
|
||||
|
||||
class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
|
||||
download_directory, key, stream_name, file_name):
|
||||
super().__init__(stream_hash, peer_finder, rate_limiter,
|
||||
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
|
||||
payment_rate_manager, wallet, download_directory, key, stream_name, file_name):
|
||||
super().__init__(conf, stream_hash, peer_finder, rate_limiter,
|
||||
blob_manager, storage, payment_rate_manager,
|
||||
wallet, key, stream_name, file_name)
|
||||
self.download_directory = unhexlify(download_directory).decode()
|
||||
@@ -147,10 +144,6 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
def __str__(self):
|
||||
return str(self.file_written_to)
|
||||
|
||||
def stop(self, err=None):
|
||||
d = EncryptedFileDownloader.stop(self, err=err)
|
||||
return d
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
self.blob_manager,
|
||||
@@ -173,15 +166,11 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
|
||||
def _close_output(self):
|
||||
self.file_handle, file_handle = None, self.file_handle
|
||||
|
||||
def close_file():
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
return threads.deferToThread(close_file)
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
def _get_write_func(self):
|
||||
def write_func(data):
|
||||
@@ -191,8 +180,8 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
|
||||
|
||||
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
|
||||
super().__init__(peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
|
||||
super().__init__(conf, peer_finder, rate_limiter, blob_manager, storage, wallet)
|
||||
self.download_directory = hexlify(download_directory.encode())
|
||||
|
||||
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
|
||||
@@ -200,8 +189,9 @@ class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
|
||||
key = stream_info.raw_info['key']
|
||||
suggested_file_name = stream_info.raw_info['suggested_file_name']
|
||||
return EncryptedFileSaver(
|
||||
stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage, payment_rate_manager,
|
||||
self.wallet, self.download_directory, key=key, stream_name=stream_name, file_name=suggested_file_name
|
||||
self.conf, stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage,
|
||||
payment_rate_manager, self.wallet, self.download_directory, key=key, stream_name=stream_name,
|
||||
file_name=suggested_file_name
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -17,9 +18,8 @@ class EncryptedFileMetadataHandler:
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_initial_blobs(self):
|
||||
blob_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
formatted_infos = self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
defer.returnValue(formatted_infos)
|
||||
blob_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
return self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
|
||||
def final_blob_num(self):
|
||||
return self._final_blob_num
|
||||
|
||||
+455
-600
File diff suppressed because it is too large.
Load diff
@@ -1,109 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import inspect
|
||||
import logging
|
||||
TRACE = 5
|
||||
|
||||
|
||||
####
|
||||
# This code is copied from logging/__init__.py in the python source code
|
||||
####
|
||||
#
|
||||
# _srcfile is used when walking the stack to check when we've got the first
|
||||
# caller stack frame.
|
||||
#
|
||||
if hasattr(sys, 'frozen'): # support for py2exe
|
||||
_srcfile = "logging{}__init__{}".format(os.sep, __file__[-4:])
|
||||
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
|
||||
_srcfile = __file__[:-4] + '.py'
|
||||
else:
|
||||
_srcfile = __file__
|
||||
_srcfile = os.path.normcase(_srcfile)
|
||||
|
||||
|
||||
def findCaller(srcfile=None):
|
||||
"""Returns the filename, line number and function name of the caller"""
|
||||
srcfile = srcfile or _srcfile
|
||||
f = inspect.currentframe()
|
||||
# On some versions of IronPython, currentframe() returns None if
|
||||
# IronPython isn't run with -X:Frames.
|
||||
if f is not None:
|
||||
f = f.f_back
|
||||
rv = "(unknown file)", 0, "(unknown function)"
|
||||
while hasattr(f, "f_code"):
|
||||
co = f.f_code
|
||||
filename = os.path.normcase(co.co_filename)
|
||||
# ignore any function calls that are in this file
|
||||
if filename == srcfile:
|
||||
f = f.f_back
|
||||
continue
|
||||
rv = (filename, f.f_lineno, co.co_name)
|
||||
break
|
||||
return rv
|
||||
|
||||
|
||||
###
|
||||
|
||||
class Logger(logging.Logger):
|
||||
"""A logger that has an extra `fail` method useful for handling twisted failures."""
|
||||
|
||||
def fail(self, callback=None, *args, **kwargs):
|
||||
"""Returns a function to log a failure from an errback.
|
||||
|
||||
The returned function appends the error message and extracts
|
||||
the traceback from `err`.
|
||||
|
||||
Example usage:
|
||||
d.addErrback(log.fail(), 'This is an error message')
|
||||
|
||||
Although odd, making the method call is necessary to extract
|
||||
out useful filename and line number information; otherwise the
|
||||
reported values are from inside twisted's deferred handling
|
||||
code.
|
||||
|
||||
Args:
|
||||
callback: callable to call after making the log. The first argument
|
||||
will be the `err` from the deferred
|
||||
args: extra arguments to pass into `callback`
|
||||
|
||||
Returns: a function that takes the following arguments:
|
||||
err: twisted.python.failure.Failure
|
||||
msg: the message to log, using normal logging string iterpolation.
|
||||
msg_args: the values to substitute into `msg`
|
||||
msg_kwargs: set `level` to change from the default ERROR severity. Other
|
||||
keywoards are treated as normal log kwargs.
|
||||
"""
|
||||
fn, lno, func = findCaller()
|
||||
|
||||
def _fail(err, msg, *msg_args, **msg_kwargs):
|
||||
level = msg_kwargs.pop('level', logging.ERROR)
|
||||
msg += ": %s"
|
||||
msg_args += (err.getErrorMessage(),)
|
||||
exc_info = (err.type, err.value, err.getTracebackObject())
|
||||
record = self.makeRecord(
|
||||
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
|
||||
self.handle(record)
|
||||
if callback:
|
||||
try:
|
||||
return callback(err, *args, **kwargs)
|
||||
except Exception:
|
||||
# log.fail is almost always called within an
|
||||
# errback. If callback fails and we didn't catch
|
||||
# the exception we would need to attach a second
|
||||
# errback to deal with that, which we will almost
|
||||
# never do and then we end up with an unhandled
|
||||
# error that will get swallowed by twisted
|
||||
self.exception('Failed to run callback')
|
||||
|
||||
return _fail
|
||||
|
||||
def trace(self, msg, *args, **kwargs):
|
||||
if self.isEnabledFor(TRACE):
|
||||
self._log(TRACE, msg, args, **kwargs)
|
||||
|
||||
|
||||
def install_logger():
|
||||
current = logging.getLoggerClass()
|
||||
if current is not Logger:
|
||||
logging.setLoggerClass(Logger)
|
||||
logging.addLevelName(TRACE, 'TRACE')
|
||||
+128
-176
@@ -1,132 +1,47 @@
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import asyncio
|
||||
import argparse
|
||||
import typing
|
||||
|
||||
# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows
|
||||
# This needs to happen before anything else
|
||||
if 'win' in sys.platform:
|
||||
import certifi
|
||||
os.environ['SSL_CERT_FILE'] = certifi.where()
|
||||
|
||||
from twisted.internet import asyncioreactor
|
||||
if 'twisted.internet.reactor' not in sys.modules:
|
||||
asyncioreactor.install()
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
|
||||
# pyinstaller hooks install the default reactor before
|
||||
# any of our code runs, see kivy for similar problem:
|
||||
# https://github.com/kivy/kivy/issues/4182
|
||||
del sys.modules['twisted.internet.reactor']
|
||||
asyncioreactor.install()
|
||||
from twisted.internet import reactor
|
||||
import logging
|
||||
from aiohttp.client_exceptions import ClientConnectorError
|
||||
from requests.exceptions import ConnectionError
|
||||
from docopt import docopt
|
||||
from textwrap import dedent
|
||||
|
||||
from lbrynet import conf, log_support, __name__ as lbrynet_name
|
||||
from lbrynet.utils import check_connection, json_dumps_pretty
|
||||
import aiohttp
|
||||
from lbrynet.extras.compat import force_asyncioreactor_install
|
||||
force_asyncioreactor_install()
|
||||
|
||||
from lbrynet import log_support, __name__ as lbrynet_name, __version__ as lbrynet_version
|
||||
from lbrynet.extras.daemon.loggly_handler import get_loggly_handler
|
||||
from lbrynet.conf import Config, CLIConfig
|
||||
from lbrynet.utils import check_connection
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
from lbrynet.extras.daemon.DaemonConsole import main as daemon_console
|
||||
from lbrynet.extras.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.extras.system_info import get_platform
|
||||
|
||||
log = logging.getLogger(lbrynet_name)
|
||||
|
||||
optional_path_getter_type = typing.Optional[typing.Callable[[], str]]
|
||||
log.addHandler(logging.NullHandler())
|
||||
|
||||
|
||||
def start_daemon(settings: typing.Optional[typing.Dict] = None,
|
||||
console_output: typing.Optional[bool] = True, verbose: typing.Optional[typing.List[str]] = None,
|
||||
data_dir: typing.Optional[str] = None, wallet_dir: typing.Optional[str] = None,
|
||||
download_dir: typing.Optional[str] = None):
|
||||
|
||||
settings = settings or {}
|
||||
conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir)
|
||||
for k, v in settings.items():
|
||||
conf.settings.update({k, v}, data_types=(conf.TYPE_CLI,))
|
||||
|
||||
log_support.configure_logging(conf.settings.get_log_filename(), console_output, verbose)
|
||||
log_support.configure_loggly_handler()
|
||||
log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
if check_connection():
|
||||
daemon = Daemon()
|
||||
daemon.start_listening()
|
||||
reactor.run()
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
def display(data):
|
||||
print(json.dumps(data, indent=2))
|
||||
|
||||
|
||||
def start_daemon_with_cli_args(argv=None, data_dir: typing.Optional[str] = None,
|
||||
wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--http-auth", dest="useauth", action="store_true", default=False
|
||||
)
|
||||
parser.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
parser.add_argument(
|
||||
'--version', action="store_true",
|
||||
help='Show daemon version and quit'
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
settings = {}
|
||||
if args.useauth:
|
||||
settings['use_auth_http'] = True
|
||||
|
||||
verbose = None
|
||||
if args.verbose:
|
||||
verbose = args.verbose
|
||||
|
||||
console_output = not args.quiet
|
||||
|
||||
if args.version:
|
||||
print(json_dumps_pretty(get_platform()))
|
||||
return
|
||||
|
||||
return start_daemon(settings, console_output, verbose, data_dir, wallet_dir, download_dir)
|
||||
|
||||
|
||||
async def execute_command(method, params, data_dir: typing.Optional[str] = None,
|
||||
wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None):
|
||||
# this check if the daemon is running or not
|
||||
conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir)
|
||||
api = None
|
||||
try:
|
||||
api = await LBRYAPIClient.get_client()
|
||||
await api.status()
|
||||
except (ClientConnectorError, ConnectionError):
|
||||
if api:
|
||||
await api.session.close()
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
return 1
|
||||
|
||||
# this actually executes the method
|
||||
resp = await api.call(method, params)
|
||||
|
||||
try:
|
||||
await api.session.close()
|
||||
print(json.dumps(resp["result"], indent=2))
|
||||
except KeyError:
|
||||
if resp["error"]["code"] == -32500:
|
||||
print(json.dumps(resp["error"], indent=2))
|
||||
else:
|
||||
print(json.dumps(resp["error"]["message"], indent=2))
|
||||
async def execute_command(conf, method, params):
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
message = {'method': method, 'params': params}
|
||||
async with session.get(conf.api_connection_url, json=message) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
if 'result' in data:
|
||||
display(data['result'])
|
||||
elif 'error' in data:
|
||||
if 'message' in data['error']:
|
||||
display(data['error']['message'])
|
||||
else:
|
||||
display(data['error'])
|
||||
except Exception as e:
|
||||
log.exception('Could not process response from server:', exc_info=e)
|
||||
except aiohttp.ClientConnectionError:
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
|
||||
|
||||
def print_help():
|
||||
@@ -200,82 +115,119 @@ def set_kwargs(parsed_args):
|
||||
return kwargs
|
||||
|
||||
|
||||
def get_argument_parser():
|
||||
main = argparse.ArgumentParser('lbrynet', add_help=False)
|
||||
main.add_argument(
|
||||
'--version', dest='cli_version', action="store_true",
|
||||
help='Show lbrynet CLI version and exit.'
|
||||
)
|
||||
main.add_argument(
|
||||
'-h', '--help', dest='help', action="store_true",
|
||||
help='Show this help message and exit'
|
||||
)
|
||||
CLIConfig.contribute_args(main)
|
||||
sub = main.add_subparsers(dest='command')
|
||||
help = sub.add_parser('help', help='Detailed help for remote commands.')
|
||||
help.add_argument('help_command', nargs='*')
|
||||
start = sub.add_parser('start', help='Start lbrynet server.')
|
||||
start.add_argument(
|
||||
'--quiet', dest='quiet', action="store_true",
|
||||
help='Disable all console output.'
|
||||
)
|
||||
start.add_argument(
|
||||
'--verbose', nargs="*",
|
||||
help=('Enable debug output. Optionally specify loggers for which debug output '
|
||||
'should selectively be applied.')
|
||||
)
|
||||
Config.contribute_args(start)
|
||||
api = Daemon.get_api_definitions()
|
||||
for group in sorted(api):
|
||||
group_command = sub.add_parser(group, help=api[group]['doc'])
|
||||
group_command.set_defaults(group_doc=group_command)
|
||||
if group in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
|
||||
continue
|
||||
commands = group_command.add_subparsers(dest='subcommand')
|
||||
for command in api[group]['commands']:
|
||||
commands.add_parser(command['name'], help=command['doc'].strip().splitlines()[0])
|
||||
for deprecated in Daemon.deprecated_methods:
|
||||
group_command = sub.add_parser(deprecated)
|
||||
group_command.add_subparsers(dest='subcommand')
|
||||
return main
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv or sys.argv[1:]
|
||||
if not argv:
|
||||
print_help()
|
||||
return 1
|
||||
parser = get_argument_parser()
|
||||
args, command_args = parser.parse_known_args(argv)
|
||||
|
||||
dir_args = {}
|
||||
if len(argv) > 2:
|
||||
dir_arg_keys = [
|
||||
'data_dir',
|
||||
'wallet_dir',
|
||||
'download_directory'
|
||||
]
|
||||
conf = Config.create_from_arguments(args)
|
||||
|
||||
for arg in argv:
|
||||
found_dir_arg = False
|
||||
for key in dir_arg_keys:
|
||||
if arg.startswith(f'--{key}='):
|
||||
if key in dir_args:
|
||||
print(f"Multiple values provided for '{key}' argument")
|
||||
print_help()
|
||||
return 1
|
||||
dir_args[key] = os.path.expanduser(os.path.expandvars(arg.lstrip(f'--{key}=')))
|
||||
found_dir_arg = True
|
||||
if not found_dir_arg:
|
||||
break
|
||||
argv = argv[len(dir_args):]
|
||||
if args.cli_version:
|
||||
print(f"{lbrynet_name} {lbrynet_version}")
|
||||
return 0
|
||||
|
||||
data_dir = dir_args.get('data_dir')
|
||||
wallet_dir = dir_args.get('wallet_dir')
|
||||
download_dir = dir_args.get('download_directory')
|
||||
elif args.command == 'start':
|
||||
|
||||
for k, v in dir_args.items():
|
||||
if not os.path.isdir(v):
|
||||
print(f"'{data_dir}' is not a directory, cannot use it for {k}")
|
||||
return 1
|
||||
log_support.configure_logging(conf.log_file_path, not args.quiet, args.verbose)
|
||||
|
||||
method, args = argv[0], argv[1:]
|
||||
if conf.share_usage_data:
|
||||
loggly_handler = get_loggly_handler()
|
||||
loggly_handler.setLevel(logging.ERROR)
|
||||
log.addHandler(loggly_handler)
|
||||
|
||||
if method in ['help', '--help', '-h']:
|
||||
if len(args) == 1:
|
||||
print_help_for_command(args[0])
|
||||
log.debug('Final Settings: %s', conf.settings_dict)
|
||||
log.info("Starting lbrynet-daemon from command line")
|
||||
|
||||
daemon = Daemon(conf)
|
||||
|
||||
if check_connection():
|
||||
from twisted.internet import reactor
|
||||
reactor._asyncioEventloop.create_task(daemon.start())
|
||||
reactor.run()
|
||||
else:
|
||||
print_help()
|
||||
return 0
|
||||
log.info("Not connected to internet, unable to start")
|
||||
|
||||
elif method in ['version', '--version', '-v']:
|
||||
print("{lbrynet_name} {lbrynet_version}".format(
|
||||
lbrynet_name=lbrynet_name, **get_platform()
|
||||
))
|
||||
return 0
|
||||
elif args.command == 'help':
|
||||
|
||||
elif method == 'start':
|
||||
sys.exit(start_daemon_with_cli_args(args, data_dir, wallet_dir, download_dir))
|
||||
|
||||
elif method == 'console':
|
||||
sys.exit(daemon_console())
|
||||
|
||||
elif method not in Daemon.callable_methods:
|
||||
if method not in Daemon.deprecated_methods:
|
||||
print(f'{method} is not a valid command.')
|
||||
return 1
|
||||
|
||||
new_method = Daemon.deprecated_methods[method].new_command
|
||||
if new_method is None:
|
||||
print(f"{method} is permanently deprecated and does not have a replacement command.")
|
||||
if args.help_command:
|
||||
method = '_'.join(args.help_command)
|
||||
else:
|
||||
parser.print_help()
|
||||
return 0
|
||||
|
||||
print(f"{method} is deprecated, using {new_method}.")
|
||||
method = new_method
|
||||
if method not in Daemon.callable_methods:
|
||||
print('Invalid command name: {method}')
|
||||
return 1
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
parsed = docopt(fn.__doc__, args)
|
||||
params = set_kwargs(parsed)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(execute_command(method, params, data_dir, wallet_dir, download_dir))
|
||||
fn = Daemon.callable_methods[method]
|
||||
print(fn.__doc__)
|
||||
|
||||
elif args.command is not None:
|
||||
|
||||
if args.command in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
|
||||
method = args.command
|
||||
elif args.subcommand is not None:
|
||||
method = f'{args.command}_{args.subcommand}'
|
||||
else:
|
||||
args.group_doc.print_help()
|
||||
return 0
|
||||
|
||||
if method in Daemon.deprecated_methods:
|
||||
new_method = Daemon.deprecated_methods[method].new_command
|
||||
if new_method is None:
|
||||
print(f"{method} is permanently deprecated and does not have a replacement command.")
|
||||
return 0
|
||||
print(f"{method} is deprecated, using {new_method}.")
|
||||
method = new_method
|
||||
|
||||
fn = Daemon.callable_methods[method]
|
||||
parsed = docopt(fn.__doc__, command_args)
|
||||
params = set_kwargs(parsed)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(execute_command(conf, method, params))
|
||||
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import asyncio
|
||||
|
||||
|
||||
def force_asyncioreactor_install():
|
||||
import sys
|
||||
from twisted.internet import asyncioreactor
|
||||
if 'twisted.internet.reactor' not in sys.modules:
|
||||
asyncioreactor.install()
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
|
||||
# pyinstaller hooks install the default reactor before
|
||||
# any of our code runs, see kivy for similar problem:
|
||||
# https://github.com/kivy/kivy/issues/4182
|
||||
del sys.modules['twisted.internet.reactor']
|
||||
asyncioreactor.install()
|
||||
|
||||
|
||||
def d2f(deferred):
|
||||
return deferred.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def f2d(future):
|
||||
from twisted.internet import defer
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
@@ -1,6 +1,8 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from twisted._threads import AlreadyQuit
|
||||
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -26,6 +28,7 @@ class Component(metaclass=ComponentType):
|
||||
component_name = None
|
||||
|
||||
def __init__(self, component_manager):
|
||||
self.conf: Config = component_manager.conf
|
||||
self.component_manager = component_manager
|
||||
self._running = False
|
||||
|
||||
@@ -36,37 +39,35 @@ class Component(metaclass=ComponentType):
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup(self):
|
||||
async def _setup(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.start)
|
||||
result = await self.start()
|
||||
self._running = True
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||
log.exception("Error setting up %s", self.component_name or self.__class__.__name__)
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop(self):
|
||||
async def _stop(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.stop)
|
||||
result = await self.stop()
|
||||
self._running = False
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from lbrynet.p2p.Error import ComponentStartConditionNotMet
|
||||
from lbrynet.extras.daemon.PeerManager import PeerManager
|
||||
from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
|
||||
from lbrynet.conf import Config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -34,8 +35,9 @@ class RequiredCondition(metaclass=RequiredConditionType):
|
||||
class ComponentManager:
|
||||
default_component_classes = {}
|
||||
|
||||
def __init__(self, reactor=None, analytics_manager=None, skip_components=None,
|
||||
def __init__(self, conf: Config, reactor=None, analytics_manager=None, skip_components=None,
|
||||
peer_manager=None, peer_finder=None, **override_components):
|
||||
self.conf = conf
|
||||
self.skip_components = skip_components or []
|
||||
self.reactor = reactor
|
||||
self.component_classes = {}
|
||||
@@ -55,6 +57,7 @@ class ComponentManager:
|
||||
|
||||
for component_class in self.component_classes.values():
|
||||
self.components.add(component_class(self))
|
||||
self.daemon = None
|
||||
|
||||
def evaluate_condition(self, condition_name):
|
||||
if condition_name not in RegisteredConditions.conditions:
|
||||
@@ -110,13 +113,8 @@ class ComponentManager:
|
||||
steps.reverse()
|
||||
return steps
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self, **callbacks):
|
||||
"""
|
||||
Start Components in sequence sorted by requirements
|
||||
|
||||
:return: (defer.Deferred)
|
||||
"""
|
||||
async def setup(self, **callbacks):
|
||||
""" Start Components in sequence sorted by requirements """
|
||||
for component_name, cb in callbacks.items():
|
||||
if component_name not in self.component_classes:
|
||||
if component_name not in self.skip_components:
|
||||
@@ -124,19 +122,22 @@ class ComponentManager:
|
||||
if not callable(cb):
|
||||
raise ValueError("%s is not callable" % cb)
|
||||
|
||||
def _setup(component):
|
||||
async def _setup(component):
|
||||
await component._setup()
|
||||
if component.component_name in callbacks:
|
||||
d = component._setup()
|
||||
d.addCallback(callbacks[component.component_name], component)
|
||||
return d
|
||||
return component._setup()
|
||||
maybe_coro = callbacks[component.component_name](component)
|
||||
if asyncio.iscoroutine(maybe_coro):
|
||||
asyncio.create_task(maybe_coro)
|
||||
|
||||
stages = self.sort_components()
|
||||
for stage in stages:
|
||||
yield defer.DeferredList([_setup(component) for component in stage if not component.running])
|
||||
needing_start = [
|
||||
_setup(component) for component in stage if not component.running
|
||||
]
|
||||
if needing_start:
|
||||
await asyncio.wait(needing_start)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
"""
|
||||
Stop Components in reversed startup order
|
||||
|
||||
@@ -144,7 +145,11 @@ class ComponentManager:
|
||||
"""
|
||||
stages = self.sort_components(reverse=True)
|
||||
for stage in stages:
|
||||
yield defer.DeferredList([component._stop() for component in stage if component.running])
|
||||
needing_stop = [
|
||||
component._stop() for component in stage if component.running
|
||||
]
|
||||
if needing_stop:
|
||||
await asyncio.wait(needing_stop)
|
||||
|
||||
def all_components_running(self, *component_names):
|
||||
"""
|
||||
|
||||
+139
-178
@@ -1,33 +1,34 @@
|
||||
import os
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import logging
|
||||
import treq
|
||||
import json
|
||||
import math
|
||||
import binascii
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
from twisted.internet import defer, threads, reactor, error, task
|
||||
from twisted.internet import defer, reactor, error
|
||||
|
||||
from aioupnp import __version__ as aioupnp_version
|
||||
from aioupnp.upnp import UPnP
|
||||
from aioupnp.fault import UPnPError
|
||||
|
||||
import lbrynet.schema
|
||||
from lbrynet import conf
|
||||
|
||||
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
|
||||
from lbrynet.extras.compat import d2f
|
||||
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
|
||||
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.Component import Component
|
||||
from lbrynet.extras.daemon.ExchangeRateManager import ExchangeRateManager
|
||||
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
|
||||
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.wallet import Network
|
||||
from lbrynet.utils import DeferredDict, generate_id
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
|
||||
from lbrynet.p2p.RateLimiter import RateLimiter
|
||||
from lbrynet.p2p.BlobManager import DiskBlobManager
|
||||
@@ -55,22 +56,23 @@ RATE_LIMITER_COMPONENT = "rate_limiter"
|
||||
PAYMENT_RATE_COMPONENT = "payment_rate_manager"
|
||||
|
||||
|
||||
def from_future(coroutine: asyncio.coroutine) -> defer.Deferred:
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(coroutine))
|
||||
async def gather_dict(tasks: dict):
|
||||
async def wait_value(key, value):
|
||||
return key, await value
|
||||
return dict(await asyncio.gather(*(
|
||||
wait_value(*kv) for kv in tasks.items()
|
||||
)))
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
async def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
buf = []
|
||||
response = yield treq.get("https://api.lbry.io/ip")
|
||||
yield treq.collect(response, buf.append)
|
||||
parsed = json.loads(b"".join(buf).decode())
|
||||
if parsed['success']:
|
||||
return parsed['data']['ip']
|
||||
return
|
||||
except Exception as err:
|
||||
return
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get("https://api.lbry.io/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
@@ -86,28 +88,26 @@ class DatabaseComponent(Component):
|
||||
|
||||
@staticmethod
|
||||
def get_current_db_revision():
|
||||
return 10
|
||||
return 9
|
||||
|
||||
@staticmethod
|
||||
def get_revision_filename():
|
||||
return conf.settings.get_db_revision_filename()
|
||||
@property
|
||||
def revision_filename(self):
|
||||
return self.component_manager.daemon.db_revision_file_path
|
||||
|
||||
@staticmethod
|
||||
def _write_db_revision_file(version_num):
|
||||
with open(conf.settings.get_db_revision_filename(), mode='w') as db_revision:
|
||||
def _write_db_revision_file(self, version_num):
|
||||
with open(self.revision_filename, mode='w') as db_revision:
|
||||
db_revision.write(str(version_num))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
# check directories exist, create them if they don't
|
||||
log.info("Loading databases")
|
||||
|
||||
if not os.path.exists(self.get_revision_filename()):
|
||||
if not os.path.exists(self.revision_filename):
|
||||
log.warning("db_revision file not found. Creating it")
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
|
||||
# check the db migration and run any needed migrations
|
||||
with open(self.get_revision_filename(), "r") as revision_read_handle:
|
||||
with open(self.revision_filename, "r") as revision_read_handle:
|
||||
old_revision = int(revision_read_handle.read().strip())
|
||||
|
||||
if old_revision > self.get_current_db_revision():
|
||||
@@ -117,19 +117,19 @@ class DatabaseComponent(Component):
|
||||
if old_revision < self.get_current_db_revision():
|
||||
from lbrynet.extras.daemon.migrator import dbmigrator
|
||||
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
|
||||
yield threads.deferToThread(
|
||||
dbmigrator.migrate_db, conf.settings.data_dir, old_revision, self.get_current_db_revision()
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
|
||||
)
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.info("Finished upgrading the databases.")
|
||||
|
||||
# start SQLiteStorage
|
||||
self.storage = SQLiteStorage(conf.settings.data_dir)
|
||||
yield self.storage.setup()
|
||||
self.storage = SQLiteStorage(
|
||||
self.conf, os.path.join(self.conf.data_dir, "lbrynet.sqlite")
|
||||
)
|
||||
await self.storage.open()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.storage.stop()
|
||||
async def stop(self):
|
||||
await self.storage.close()
|
||||
self.storage = None
|
||||
|
||||
|
||||
@@ -142,9 +142,9 @@ class HeadersComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.headers_dir = os.path.join(conf.settings.wallet_dir, 'lbc_mainnet')
|
||||
self.headers_dir = os.path.join(self.conf.wallet_dir, 'lbc_mainnet')
|
||||
self.headers_file = os.path.join(self.headers_dir, 'headers')
|
||||
self.old_file = os.path.join(conf.settings.wallet_dir, 'blockchain_headers')
|
||||
self.old_file = os.path.join(self.conf.wallet_dir, 'blockchain_headers')
|
||||
self._downloading_headers = None
|
||||
self._headers_progress_percent = 0
|
||||
|
||||
@@ -152,7 +152,7 @@ class HeadersComponent(Component):
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {} if not self._downloading_headers else {
|
||||
'downloading_headers': self._downloading_headers,
|
||||
'download_progress': self._headers_progress_percent
|
||||
@@ -203,8 +203,8 @@ class HeadersComponent(Component):
|
||||
async def get_remote_height(self):
|
||||
ledger = SimpleNamespace()
|
||||
ledger.config = {
|
||||
'default_servers': conf.settings['lbryum_servers'],
|
||||
'data_path': conf.settings.wallet_dir
|
||||
'default_servers': self.conf.lbryum_servers,
|
||||
'data_path': self.conf.wallet_dir
|
||||
}
|
||||
net = Network(ledger)
|
||||
first_connection = net.on_connected.first
|
||||
@@ -215,10 +215,10 @@ class HeadersComponent(Component):
|
||||
return remote_height
|
||||
|
||||
async def should_download_headers_from_s3(self):
|
||||
if conf.settings['blockchain_name'] != "lbrycrd_main":
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
return False
|
||||
self._check_header_file_integrity()
|
||||
s3_headers_depth = conf.settings['s3_headers_depth']
|
||||
s3_headers_depth = self.conf.s3_headers_depth
|
||||
if not s3_headers_depth:
|
||||
return False
|
||||
local_height = self.local_header_file_height()
|
||||
@@ -230,10 +230,10 @@ class HeadersComponent(Component):
|
||||
|
||||
def _check_header_file_integrity(self):
|
||||
# TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
|
||||
if conf.settings['blockchain_name'] != "lbrycrd_main":
|
||||
if self.conf.blockchain_name != "lbrycrd_main":
|
||||
return
|
||||
hashsum = sha256()
|
||||
checksum_height, checksum = conf.settings['HEADERS_FILE_SHA256_CHECKSUM']
|
||||
checksum_height, checksum = HEADERS_FILE_SHA256_CHECKSUM
|
||||
checksum_length_in_bytes = checksum_height * HEADER_SIZE
|
||||
if self.local_header_file_size() < checksum_length_in_bytes:
|
||||
return
|
||||
@@ -250,33 +250,23 @@ class HeadersComponent(Component):
|
||||
with open(self.headers_file, "rb+") as headers_file:
|
||||
headers_file.truncate(checksum_length_in_bytes)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
conf.settings.ensure_wallet_dir()
|
||||
async def start(self):
|
||||
if not os.path.exists(self.headers_dir):
|
||||
os.mkdir(self.headers_dir)
|
||||
if os.path.exists(self.old_file):
|
||||
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
|
||||
os.rename(self.old_file, self.headers_file)
|
||||
self._downloading_headers = yield f2d(self.should_download_headers_from_s3())
|
||||
self._downloading_headers = await self.should_download_headers_from_s3()
|
||||
if self._downloading_headers:
|
||||
try:
|
||||
yield self.fetch_headers_from_s3()
|
||||
await d2f(self.fetch_headers_from_s3())
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self._downloading_headers = False
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
def d2f(deferred):
|
||||
return deferred.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def f2d(future):
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class WalletComponent(Component):
|
||||
@@ -291,7 +281,7 @@ class WalletComponent(Component):
|
||||
def component(self):
|
||||
return self.wallet_manager
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
if self.wallet_manager and self.running:
|
||||
local_height = self.wallet_manager.network.get_local_height()
|
||||
remote_height = self.wallet_manager.network.get_server_height()
|
||||
@@ -304,19 +294,16 @@ class WalletComponent(Component):
|
||||
'is_locked': not self.wallet_manager.is_wallet_unlocked,
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
conf.settings.ensure_wallet_dir()
|
||||
async def start(self):
|
||||
log.info("Starting torba wallet")
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = conf.settings['blockchain_name']
|
||||
self.wallet_manager = yield f2d(LbryWalletManager.from_lbrynet_config(conf.settings, storage))
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = self.conf.blockchain_name
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf, storage)
|
||||
self.wallet_manager.old_db = storage
|
||||
yield f2d(self.wallet_manager.start())
|
||||
await self.wallet_manager.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield f2d(self.wallet_manager.stop())
|
||||
async def stop(self):
|
||||
await self.wallet_manager.stop()
|
||||
self.wallet_manager = None
|
||||
|
||||
|
||||
@@ -339,20 +326,17 @@ class BlobComponent(Component):
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
if dht_node:
|
||||
datastore = dht_node._dataStore
|
||||
self.blob_manager = DiskBlobManager(os.path.join(conf.settings.data_dir, "blobfiles"), storage, datastore)
|
||||
self.blob_manager = DiskBlobManager(os.path.join(self.conf.data_dir, "blobfiles"), storage, datastore)
|
||||
return self.blob_manager.setup()
|
||||
|
||||
def stop(self):
|
||||
return self.blob_manager.stop()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
count = 0
|
||||
if self.blob_manager:
|
||||
count = yield self.blob_manager.storage.count_finished_blobs()
|
||||
defer.returnValue({
|
||||
'finished_blobs': count
|
||||
})
|
||||
count = await self.blob_manager.storage.count_finished_blobs()
|
||||
return {'finished_blobs': count}
|
||||
|
||||
|
||||
class DHTComponent(Component):
|
||||
@@ -370,41 +354,39 @@ class DHTComponent(Component):
|
||||
def component(self):
|
||||
return self.dht_node
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {
|
||||
'node_id': binascii.hexlify(conf.settings.get_node_id()),
|
||||
'node_id': binascii.hexlify(self.component_manager.daemon.node_id),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", conf.settings["peer_port"])
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", conf.settings["dht_node_port"])
|
||||
node_id = conf.settings.get_node_id()
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
|
||||
node_id = self.component_manager.daemon.node_id
|
||||
if node_id is None:
|
||||
node_id = generate_id()
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
external_ip = yield get_external_ip()
|
||||
external_ip = await get_external_ip()
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
self.dht_node = Node(
|
||||
node_id=node_id,
|
||||
udpPort=conf.settings['dht_node_port'],
|
||||
udpPort=self.conf.dht_node_port,
|
||||
externalUDPPort=self.external_udp_port,
|
||||
externalIP=external_ip,
|
||||
peerPort=self.external_peer_port
|
||||
)
|
||||
|
||||
yield self.dht_node.start(conf.settings['known_dht_nodes'], block_on_join=False)
|
||||
await d2f(self.dht_node.start(self.conf.known_dht_nodes, block_on_join=False))
|
||||
log.info("Started the dht")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.dht_node.stop()
|
||||
return d2f(self.dht_node.stop())
|
||||
|
||||
|
||||
class HashAnnouncerComponent(Component):
|
||||
@@ -419,18 +401,16 @@ class HashAnnouncerComponent(Component):
|
||||
def component(self):
|
||||
return self.hash_announcer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = DHTHashAnnouncer(dht_node, storage)
|
||||
yield self.hash_announcer.start()
|
||||
self.hash_announcer = DHTHashAnnouncer(self.conf, dht_node, storage)
|
||||
self.hash_announcer.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.hash_announcer.stop()
|
||||
self.hash_announcer.stop()
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {
|
||||
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.hash_queue)
|
||||
}
|
||||
@@ -447,13 +427,11 @@ class RateLimiterComponent(Component):
|
||||
def component(self):
|
||||
return self.rate_limiter
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
self.rate_limiter.start()
|
||||
return defer.succeed(None)
|
||||
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
self.rate_limiter.stop()
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class PaymentRateComponent(Component):
|
||||
@@ -467,11 +445,11 @@ class PaymentRateComponent(Component):
|
||||
def component(self):
|
||||
return self.payment_rate_manager
|
||||
|
||||
def start(self):
|
||||
return defer.succeed(None)
|
||||
async def start(self):
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class FileManagerComponent(Component):
|
||||
@@ -487,14 +465,13 @@ class FileManagerComponent(Component):
|
||||
def component(self):
|
||||
return self.file_manager
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
if not self.file_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.file_manager.lbry_files)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
@@ -504,25 +481,26 @@ class FileManagerComponent(Component):
|
||||
sd_identifier = StreamDescriptorIdentifier()
|
||||
add_lbry_file_to_sd_identifier(sd_identifier)
|
||||
file_saver_factory = EncryptedFileSaverFactory(
|
||||
self.conf,
|
||||
self.component_manager.peer_finder,
|
||||
rate_limiter,
|
||||
blob_manager,
|
||||
storage,
|
||||
wallet,
|
||||
conf.settings.download_dir
|
||||
self.conf.download_dir
|
||||
)
|
||||
yield sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
|
||||
sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
|
||||
|
||||
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
|
||||
log.info('Starting the file manager')
|
||||
self.file_manager = EncryptedFileManager(self.component_manager.peer_finder, rate_limiter, blob_manager, wallet,
|
||||
payment_rate_manager, storage, sd_identifier)
|
||||
yield self.file_manager.setup()
|
||||
log.info('Done setting up file manager')
|
||||
self.file_manager = EncryptedFileManager(
|
||||
self.conf, self.component_manager.peer_finder, rate_limiter,
|
||||
blob_manager, wallet, payment_rate_manager, storage, sd_identifier
|
||||
)
|
||||
return self.file_manager.setup()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.file_manager.stop()
|
||||
return d2f(self.file_manager.stop())
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -538,11 +516,10 @@ class PeerProtocolServerComponent(Component):
|
||||
def component(self):
|
||||
return self.lbry_server_port
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
peer_port = conf.settings['peer_port']
|
||||
peer_port = self.conf.peer_port
|
||||
query_handlers = {
|
||||
handler.get_primary_query_identifier(): handler for handler in [
|
||||
BlobRequestHandlerFactory(
|
||||
@@ -562,7 +539,7 @@ class PeerProtocolServerComponent(Component):
|
||||
try:
|
||||
log.info("Peer protocol listening on TCP %i (ext port %i)", peer_port,
|
||||
upnp.upnp_redirects.get("TCP", peer_port))
|
||||
self.lbry_server_port = yield reactor.listenTCP(peer_port, server_factory)
|
||||
self.lbry_server_port = reactor.listenTCP(peer_port, server_factory)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. Visit lbry.io/faq/how-to-change-port for"
|
||||
@@ -570,12 +547,11 @@ class PeerProtocolServerComponent(Component):
|
||||
log.error("%s", traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running on your computer." % str(e))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
if self.lbry_server_port is not None:
|
||||
self.lbry_server_port, old_port = None, self.lbry_server_port
|
||||
log.info('Stop listening on port %s', old_port.port)
|
||||
yield old_port.stopListening()
|
||||
await d2f(old_port.stopListening())
|
||||
|
||||
|
||||
class ReflectorComponent(Component):
|
||||
@@ -584,32 +560,30 @@ class ReflectorComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self.reflector_server_port = conf.settings['reflector_port']
|
||||
self.reflector_server_port = self.conf.reflector_port
|
||||
self.reflector_server = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self.reflector_server
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
log.info("Starting reflector server")
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
|
||||
reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager)
|
||||
try:
|
||||
self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory)
|
||||
self.reflector_server = await d2f(reactor.listenTCP(self.reflector_server_port, reflector_factory))
|
||||
log.info('Started reflector on port %s', self.reflector_server_port)
|
||||
except error.CannotListenError as e:
|
||||
log.exception("Couldn't bind reflector to port %d", self.reflector_server_port)
|
||||
raise ValueError(f"{e} lbrynet may already be running on your computer.")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
if self.reflector_server is not None:
|
||||
log.info("Stopping reflector server")
|
||||
self.reflector_server, p = None, self.reflector_server
|
||||
yield p.stopListening
|
||||
await d2f(p.stopListening())
|
||||
|
||||
|
||||
class UPnPComponent(Component):
|
||||
@@ -617,35 +591,29 @@ class UPnPComponent(Component):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
super().__init__(component_manager)
|
||||
self._int_peer_port = conf.settings['peer_port']
|
||||
self._int_dht_node_port = conf.settings['dht_node_port']
|
||||
self.use_upnp = conf.settings['use_upnp']
|
||||
self._int_peer_port = self.conf.peer_port
|
||||
self._int_dht_node_port = self.conf.dht_node_port
|
||||
self.use_upnp = self.conf.use_upnp
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
self.external_ip = None
|
||||
self._maintain_redirects_lc = task.LoopingCall(self._maintain_redirects)
|
||||
self._maintain_redirects_lc.clock = self.component_manager.reactor
|
||||
self._maintain_redirects_task = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup_redirects(self):
|
||||
d = {}
|
||||
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
|
||||
upnp_redirects = yield DeferredDict(d)
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
async def _repeatedly_maintain_redirects(self, now=True):
|
||||
while True:
|
||||
if now:
|
||||
await self._maintain_redirects()
|
||||
await asyncio.sleep(360)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _maintain_redirects(self):
|
||||
async def _maintain_redirects(self):
|
||||
# setup the gateway if necessary
|
||||
if not self.upnp:
|
||||
try:
|
||||
self.upnp = yield from_future(UPnP.discover())
|
||||
self.upnp = await UPnP.discover()
|
||||
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
except Exception as err:
|
||||
log.warning("upnp discovery failed: %s", err)
|
||||
@@ -655,7 +623,7 @@ class UPnPComponent(Component):
|
||||
external_ip = None
|
||||
if self.upnp:
|
||||
try:
|
||||
external_ip = yield from_future(self.upnp.get_external_ip())
|
||||
external_ip = await self.upnp.get_external_ip()
|
||||
if external_ip != "0.0.0.0" and not self.external_ip:
|
||||
log.info("got external ip from UPnP: %s", external_ip)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
@@ -663,7 +631,7 @@ class UPnPComponent(Component):
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
external_ip = yield get_external_ip()
|
||||
external_ip = await get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
self.external_ip = external_ip
|
||||
@@ -674,10 +642,10 @@ class UPnPComponent(Component):
|
||||
log.info("add UPnP port mappings")
|
||||
d = {}
|
||||
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
|
||||
d["TCP"] = self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
|
||||
upnp_redirects = yield DeferredDict(d)
|
||||
d["UDP"] = self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
upnp_redirects = await gather_dict(d)
|
||||
log.info("set up redirects: %s", upnp_redirects)
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
@@ -685,7 +653,7 @@ class UPnPComponent(Component):
|
||||
return self._maintain_redirects()
|
||||
elif self.upnp: # check existing redirects are still active
|
||||
found = set()
|
||||
mappings = yield from_future(self.upnp.get_redirects())
|
||||
mappings = await self.upnp.get_redirects()
|
||||
for mapping in mappings:
|
||||
proto = mapping['NewProtocol']
|
||||
if proto in self.upnp_redirects and mapping['NewExternalPort'] == self.upnp_redirects[proto]:
|
||||
@@ -693,18 +661,14 @@ class UPnPComponent(Component):
|
||||
found.add(proto)
|
||||
if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
udp_port = yield from_future(
|
||||
self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
)
|
||||
udp_port = await self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
self.upnp_redirects['UDP'] = udp_port
|
||||
log.info("refreshed upnp redirect for dht port: %i", udp_port)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
del self.upnp_redirects['UDP']
|
||||
if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
tcp_port = yield from_future(
|
||||
self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
)
|
||||
tcp_port = await self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
self.upnp_redirects['TCP'] = tcp_port
|
||||
log.info("refreshed upnp redirect for peer port: %i", tcp_port)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
@@ -715,14 +679,13 @@ class UPnPComponent(Component):
|
||||
if self.upnp_redirects:
|
||||
log.debug("upnp redirects are still active")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = yield get_external_ip()
|
||||
self.external_ip = await get_external_ip()
|
||||
return
|
||||
success = False
|
||||
yield self._maintain_redirects()
|
||||
await self._maintain_redirects()
|
||||
if self.upnp:
|
||||
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
|
||||
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
|
||||
@@ -733,18 +696,18 @@ class UPnPComponent(Component):
|
||||
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, self.get_status())
|
||||
self._maintain_redirects_lc.start(360, now=False)
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
|
||||
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
|
||||
|
||||
def stop(self):
|
||||
if self._maintain_redirects_lc.running:
|
||||
self._maintain_redirects_lc.stop()
|
||||
return defer.DeferredList(
|
||||
[from_future(self.upnp.delete_port_mapping(port, protocol))
|
||||
for protocol, port in self.upnp_redirects.items()]
|
||||
)
|
||||
async def stop(self):
|
||||
if self.upnp_redirects:
|
||||
await asyncio.wait([
|
||||
self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
|
||||
])
|
||||
if self._maintain_redirects_task is not None and not self._maintain_redirects_task.done():
|
||||
self._maintain_redirects_task.cancel()
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {
|
||||
'aioupnp_version': aioupnp_version,
|
||||
'redirects': self.upnp_redirects,
|
||||
@@ -766,10 +729,8 @@ class ExchangeRateManagerComponent(Component):
|
||||
def component(self):
|
||||
return self.exchange_rate_manager
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
yield self.exchange_rate_manager.start()
|
||||
async def start(self):
|
||||
self.exchange_rate_manager.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.exchange_rate_manager.stop()
|
||||
async def stop(self):
|
||||
self.exchange_rate_manager.stop()
|
||||
+687
-425
File diff suppressed because it is too large.
Load diff
@@ -1,230 +0,0 @@
|
||||
import sys
|
||||
import code
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging.handlers
|
||||
from twisted.internet import defer, reactor, threads
|
||||
from aiohttp import client_exceptions
|
||||
|
||||
from lbrynet import utils, conf, log_support
|
||||
from lbrynet.extras.daemon import analytics
|
||||
from lbrynet.extras.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
if sys.platform.startswith('darwin') or sys.platform.startswith('linux'):
|
||||
def color(msg, c="white"):
|
||||
_colors = {
|
||||
"normal": (0, 37),
|
||||
"underlined": (2, 37),
|
||||
"red": (1, 31),
|
||||
"green": (1, 32),
|
||||
"yellow": (1, 33),
|
||||
"blue": (1, 33),
|
||||
"magenta": (1, 34),
|
||||
"cyan": (1, 35),
|
||||
"white": (1, 36),
|
||||
"grey": (1, 37)
|
||||
}
|
||||
i, j = _colors[c]
|
||||
return "\033[%i;%i;40m%s\033[0m" % (i, j, msg)
|
||||
|
||||
|
||||
logo = """\
|
||||
╓▄█▄ç
|
||||
,▄█▓▓▀▀▀▓▓▓▌▄,
|
||||
▄▄▓▓▓▀¬ ╙▀█▓▓▓▄▄
|
||||
,▄█▓▓▀▀ ^▀▀▓▓▓▌▄,
|
||||
▄█▓▓█▀` ╙▀█▓▓▓▄▄
|
||||
╓▄▓▓▓▀╙ ▀▀▓▓▓▌▄,
|
||||
▄█▓▓█▀ ╙▀▓▓
|
||||
╓▄▓▓▓▀` ▄█▓▓▓▀
|
||||
▓▓█▀ ,▄▓▓▓▀╙
|
||||
▓▓m ╟▌▄, ▄█▓▓█▀ ,,╓µ
|
||||
▓▓m ^▀█▓▓▓▄▄ ╓▄▓▓▓▀╙ █▓▓▓▓▓▀
|
||||
▓▓Q '▀▀▓▓▓▌▄, ,▄█▓▓█▀ ▄█▓▓▓▓▓▀
|
||||
▀▓▓▓▌▄, ╙▀█▓▓█▄╗ ╓▄▓▓▓▀ ╓▄▓▓▓▀▀ ▀▀
|
||||
╙▀█▓▓█▄╗ ^▀▀▓▓▓▌▄▄█▓▓▀▀ ▄█▓▓█▀`
|
||||
'▀▀▓▓▓▌▄, ╙▀██▀` ╓▄▓▓▓▀╙
|
||||
╙▀█▓▓█▄╥ ,▄█▓▓▀▀
|
||||
└▀▀▓▓▓▌▄ ▄▒▓▓▓▀╙
|
||||
╙▀█▓▓█▓▓▓▀▀
|
||||
╙▀▀`
|
||||
|
||||
"""
|
||||
else:
|
||||
def color(msg, c=None):
|
||||
return msg
|
||||
|
||||
logo = """\
|
||||
'.
|
||||
++++.
|
||||
+++,;+++,
|
||||
:+++ :+++:
|
||||
+++ ,+++;
|
||||
'++; .+++'
|
||||
`+++ `++++
|
||||
+++. `++++
|
||||
;+++ ++++
|
||||
+++ +++
|
||||
+++: '+
|
||||
,+++ +++
|
||||
+++` +++:
|
||||
`+' ,+++
|
||||
`+ + +++
|
||||
`+ +++ '++' :'+++:
|
||||
`+ ++++ `+++ ++++
|
||||
`+ ++++ +++. :+++'
|
||||
`+, ++++ ;+++ +++++
|
||||
`+++, ++++ +++ +++; +
|
||||
,+++, ++++ +++: .+++
|
||||
,+++: '++++++ +++`
|
||||
,+++: '++ '++'
|
||||
,+++: `+++
|
||||
.+++; +++,
|
||||
.+++; ;+++
|
||||
.+++; +++
|
||||
`+++++:
|
||||
`++
|
||||
|
||||
"""
|
||||
|
||||
welcometext = """\
|
||||
For a list of available commands:
|
||||
>>>help()
|
||||
|
||||
To see the documentation for a given command:
|
||||
>>>help("resolve")
|
||||
|
||||
To exit:
|
||||
>>>exit()
|
||||
"""
|
||||
|
||||
welcome = "{:*^60}\n".format(" Welcome to the lbrynet interactive console! ")
|
||||
welcome += "\n".join([f"{w:<60}" for w in welcometext.splitlines()])
|
||||
welcome += "\n%s" % ("*" * 60)
|
||||
welcome = color(welcome, "grey")
|
||||
banner = color(logo, "green") + color(welcome, "grey")
|
||||
|
||||
|
||||
def get_methods(daemon):
|
||||
locs = {}
|
||||
|
||||
def wrapped(name, fn):
|
||||
client = LBRYAPIClient.get_client()
|
||||
_fn = getattr(client, name)
|
||||
_fn.__doc__ = fn.__doc__
|
||||
return {name: _fn}
|
||||
|
||||
for method_name, method in daemon.callable_methods.items():
|
||||
locs.update(wrapped(method_name, method))
|
||||
return locs
|
||||
|
||||
|
||||
def run_terminal(callable_methods, started_daemon, quiet=False):
|
||||
locs = {}
|
||||
locs.update(callable_methods)
|
||||
|
||||
def help(method_name=None):
|
||||
if not method_name:
|
||||
print("Available api functions: ")
|
||||
for name in callable_methods:
|
||||
print("\t%s" % name)
|
||||
return
|
||||
if method_name not in callable_methods:
|
||||
print("\"%s\" is not a recognized api function")
|
||||
return
|
||||
print(callable_methods[method_name].__doc__)
|
||||
return
|
||||
|
||||
locs.update({'help': help})
|
||||
|
||||
if started_daemon:
|
||||
def exit(status=None):
|
||||
if not quiet:
|
||||
print("Stopping lbrynet-daemon...")
|
||||
callable_methods['daemon_stop']()
|
||||
return sys.exit(status)
|
||||
|
||||
locs.update({'exit': exit})
|
||||
else:
|
||||
def exit(status=None):
|
||||
try:
|
||||
reactor.callLater(0, reactor.stop)
|
||||
except Exception as err:
|
||||
print(f"error stopping reactor: {err}")
|
||||
return sys.exit(status)
|
||||
|
||||
locs.update({'exit': exit})
|
||||
|
||||
code.interact(banner if not quiet else "", local=locs)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_server_and_listen(use_auth, analytics_manager, quiet):
|
||||
log_support.configure_console()
|
||||
logging.getLogger("lbrynet").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("lbryum").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("requests").setLevel(logging.CRITICAL)
|
||||
|
||||
analytics_manager.send_server_startup()
|
||||
yield Daemon().start_listening()
|
||||
|
||||
|
||||
def threaded_terminal(started_daemon, quiet):
|
||||
callable_methods = get_methods(Daemon)
|
||||
d = threads.deferToThread(run_terminal, callable_methods, started_daemon, quiet)
|
||||
d.addErrback(lambda err: err.trap(SystemExit))
|
||||
d.addErrback(log.exception)
|
||||
|
||||
|
||||
async def start_lbrynet_console(quiet, use_existing_daemon, useauth):
|
||||
if not utils.check_connection():
|
||||
print("Not connected to internet, unable to start")
|
||||
raise Exception("Not connected to internet, unable to start")
|
||||
if not quiet:
|
||||
print("Starting lbrynet-console...")
|
||||
try:
|
||||
await LBRYAPIClient.get_client().status()
|
||||
d = defer.succeed(False)
|
||||
if not quiet:
|
||||
print("lbrynet-daemon is already running, connecting to it...")
|
||||
except client_exceptions.ClientConnectorError:
|
||||
if not use_existing_daemon:
|
||||
if not quiet:
|
||||
print("Starting lbrynet-daemon...")
|
||||
analytics_manager = analytics.Manager.new_instance()
|
||||
d = start_server_and_listen(useauth, analytics_manager, quiet)
|
||||
else:
|
||||
raise Exception("cannot connect to an existing daemon instance, "
|
||||
"and set to not start a new one")
|
||||
d.addCallback(threaded_terminal, quiet)
|
||||
d.addErrback(log.exception)
|
||||
|
||||
|
||||
def main():
|
||||
conf.initialize_settings()
|
||||
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
|
||||
parser.add_argument(
|
||||
"--use_existing_daemon",
|
||||
help="Start lbrynet-daemon if it isn't already running",
|
||||
action="store_true",
|
||||
default=False,
|
||||
dest="use_existing_daemon"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--quiet", dest="quiet", action="store_true", default=False
|
||||
)
|
||||
parser.add_argument(
|
||||
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
|
||||
)
|
||||
args = parser.parse_args()
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(start_lbrynet_console(args.quiet, args.use_existing_daemon, args.useauth))
|
||||
reactor.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -2,7 +2,7 @@ import logging
|
||||
import os
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.schema.fee import Fee
|
||||
|
||||
from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
|
||||
@@ -11,7 +11,7 @@ from lbrynet.p2p.StreamDescriptor import download_sd_blob
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from torba.client.constants import COIN
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.extras.daemon.Components import f2d
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
INITIALIZING_CODE = 'initializing'
|
||||
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
|
||||
@@ -31,17 +31,18 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GetStream:
|
||||
def __init__(self, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder, rate_limiter,
|
||||
payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None, timeout=None,
|
||||
reactor=None):
|
||||
def __init__(self, conf: Config, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder,
|
||||
rate_limiter, payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None,
|
||||
timeout=None, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.conf = conf
|
||||
self.reactor = reactor
|
||||
self.timeout = timeout or conf.settings['download_timeout']
|
||||
self.data_rate = data_rate or conf.settings['data_rate']
|
||||
self.max_key_fee = max_key_fee or conf.settings['max_key_fee'][1]
|
||||
self.disable_max_key_fee = disable_max_key_fee or conf.settings['disable_max_key_fee']
|
||||
self.download_directory = conf.settings.download_dir
|
||||
self.timeout = timeout or conf.download_timeout
|
||||
self.data_rate = data_rate or conf.data_rate
|
||||
self.max_key_fee = max_key_fee or conf.max_key_fee
|
||||
self.disable_max_key_fee = disable_max_key_fee or conf.disable_max_key_fee
|
||||
self.download_directory = conf.download_dir
|
||||
self.timeout_counter = 0
|
||||
self.code = None
|
||||
self.sd_hash = None
|
||||
@@ -153,8 +154,8 @@ class GetStream:
|
||||
@defer.inlineCallbacks
|
||||
def _download_sd_blob(self):
|
||||
sd_blob = yield download_sd_blob(
|
||||
self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
|
||||
self.wallet, self.timeout, conf.settings['download_mirrors']
|
||||
self.conf, self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
|
||||
self.wallet, self.timeout, self.conf.download_mirrors
|
||||
)
|
||||
defer.returnValue(sd_blob)
|
||||
|
||||
@@ -162,7 +163,7 @@ class GetStream:
|
||||
def _download(self, sd_blob, name, key_fee, txid, nout, file_name=None):
|
||||
self.downloader = yield self._create_downloader(sd_blob, file_name=file_name)
|
||||
yield self.pay_key_fee(key_fee, name)
|
||||
yield self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout))
|
||||
yield f2d(self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout)))
|
||||
self.finished_deferred = self.downloader.start()
|
||||
self.downloader.download_manager.progress_manager.wrote_first_data.addCallback(
|
||||
self.data_downloading_deferred.callback
|
||||
|
||||
@@ -2,27 +2,30 @@ import binascii
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer, task
|
||||
from lbrynet import utils, conf
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DHTHashAnnouncer:
|
||||
def __init__(self, dht_node, storage, concurrent_announcers=None):
|
||||
def __init__(self, conf: Config, dht_node, storage, concurrent_announcers=None):
|
||||
self.conf = conf
|
||||
self.dht_node = dht_node
|
||||
self.storage = storage
|
||||
self.clock = dht_node.clock
|
||||
self.peer_port = dht_node.peerPort
|
||||
self.hash_queue = []
|
||||
if concurrent_announcers is None:
|
||||
self.concurrent_announcers = conf.settings['concurrent_announcers']
|
||||
self.concurrent_announcers = conf.concurrent_announcers
|
||||
else:
|
||||
self.concurrent_announcers = concurrent_announcers
|
||||
self._manage_lc = None
|
||||
if self.concurrent_announcers:
|
||||
self._manage_lc = task.LoopingCall(self.manage)
|
||||
self._manage_lc.clock = self.clock
|
||||
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.settings['concurrent_announcers'] or 1)
|
||||
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.concurrent_announcers or 1)
|
||||
|
||||
def start(self):
|
||||
if self._manage_lc:
|
||||
@@ -38,7 +41,7 @@ class DHTHashAnnouncer:
|
||||
now = self.clock.seconds()
|
||||
if storing_node_ids:
|
||||
result = (now, storing_node_ids)
|
||||
yield self.storage.update_last_announced_blob(blob_hash, now)
|
||||
yield f2d(self.storage.update_last_announced_blob(blob_hash, now))
|
||||
log.debug("Stored %s to %i peers", blob_hash[:16], len(storing_node_ids))
|
||||
else:
|
||||
result = (None, [])
|
||||
@@ -82,7 +85,7 @@ class DHTHashAnnouncer:
|
||||
if not self.dht_node.contacts:
|
||||
log.info("Not ready to start announcing hashes")
|
||||
return
|
||||
need_reannouncement = yield self.storage.get_blobs_to_announce()
|
||||
need_reannouncement = yield f2d(self.storage.get_blobs_to_announce())
|
||||
if need_reannouncement:
|
||||
yield self.immediate_announce(need_reannouncement)
|
||||
else:
|
||||
|
||||
@@ -2,7 +2,6 @@ import binascii
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer
|
||||
from lbrynet import conf
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -59,7 +58,7 @@ class DHTPeerFinder(DummyPeerFinder):
|
||||
def _execute_peer_search(self, dht_node, blob_hash, timeout):
|
||||
bin_hash = binascii.unhexlify(blob_hash)
|
||||
finished_deferred = dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
|
||||
timeout = timeout or conf.settings['peer_search_timeout']
|
||||
timeout = timeout or self.component_manager.conf.peer_search_timeout
|
||||
if timeout:
|
||||
finished_deferred.addTimeout(timeout, dht_node.clock)
|
||||
try:
|
||||
|
||||
@@ -3,7 +3,7 @@ import logging
|
||||
import os
|
||||
|
||||
from lbrynet.blob.EncryptedFileCreator import create_lbry_file
|
||||
from lbrynet.extras.daemon.mime_types import guess_mime_type
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -43,25 +43,25 @@ class Publisher:
|
||||
claim_dict['stream']['source'] = {}
|
||||
claim_dict['stream']['source']['source'] = self.lbry_file.sd_hash
|
||||
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
|
||||
claim_dict['stream']['source']['contentType'] = guess_mime_type(file_path)
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
|
||||
tx = await self.wallet.claim_name(
|
||||
self.account, name, bid, claim_dict, self.certificate, holding_address
|
||||
)
|
||||
|
||||
# check if we have a file already for this claim (if this is a publish update with a new stream)
|
||||
old_stream_hashes = await d2f(self.storage.get_old_stream_hashes_for_claim_id(
|
||||
old_stream_hashes = await self.storage.get_old_stream_hashes_for_claim_id(
|
||||
tx.outputs[0].claim_id, self.lbry_file.stream_hash
|
||||
))
|
||||
)
|
||||
if old_stream_hashes:
|
||||
for lbry_file in filter(lambda l: l.stream_hash in old_stream_hashes,
|
||||
list(self.lbry_file_manager.lbry_files)):
|
||||
await d2f(self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False))
|
||||
log.info("Removed old stream for claim update: %s", lbry_file.stream_hash)
|
||||
|
||||
await d2f(self.storage.save_content_claim(
|
||||
await self.storage.save_content_claim(
|
||||
self.lbry_file.stream_hash, tx.outputs[0].id
|
||||
))
|
||||
)
|
||||
return tx
|
||||
|
||||
async def publish_stream(self, name, bid, claim_dict, stream_hash, holding_address=None):
|
||||
@@ -70,8 +70,8 @@ class Publisher:
|
||||
self.account, name, bid, claim_dict, self.certificate, holding_address
|
||||
)
|
||||
if stream_hash: # the stream_hash returned from the db will be None if this isn't a stream we have
|
||||
await d2f(self.storage.save_content_claim(
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
))
|
||||
)
|
||||
self.lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == stream_hash][0]
|
||||
return tx
|
||||
@@ -1,11 +1,12 @@
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer, task
|
||||
import aiohttp
|
||||
|
||||
from lbrynet import conf, utils
|
||||
from lbrynet.extras import looping_call_manager, system_info
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
|
||||
from lbrynet.extras import system_info
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
@@ -27,36 +28,82 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager:
|
||||
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
|
||||
self.analytics_api = analytics_api
|
||||
|
||||
def __init__(self, conf: Config, installation_id: str, session_id: str):
|
||||
self.cookies = {}
|
||||
self.url = ANALYTICS_ENDPOINT
|
||||
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
|
||||
self._enabled = conf.share_usage_data
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.looping_call_manager = self._setup_looping_calls()
|
||||
self.context = context or self._make_context(
|
||||
system_info.get_platform(), conf.settings['wallet'])
|
||||
self.installation_id = installation_id or conf.settings.installation_id
|
||||
self.session_id = session_id or conf.settings.get_session_id()
|
||||
self.is_started = False
|
||||
self.context = self._make_context(system_info.get_platform(), 'torba')
|
||||
self.installation_id = installation_id
|
||||
self.session_id = session_id
|
||||
self.task: asyncio.Task = None
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
api = Api.new_instance(enabled)
|
||||
return cls(api)
|
||||
@property
|
||||
def is_started(self):
|
||||
return self.task is not None
|
||||
|
||||
# Things We Track
|
||||
def send_new_download_start(self, download_id, name, claim_dict):
|
||||
self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
def start(self):
|
||||
if self._enabled and self.task is None:
|
||||
self.task = asyncio.create_task(self.run())
|
||||
log.info("Start")
|
||||
|
||||
def send_new_download_success(self, download_id, name, claim_dict):
|
||||
self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
async def run(self):
|
||||
while True:
|
||||
await self._send_heartbeat()
|
||||
await asyncio.sleep(1800)
|
||||
|
||||
def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
self.task.cancel()
|
||||
|
||||
async def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
request_kwargs = {
|
||||
'method': 'POST',
|
||||
'url': self.url + endpoint,
|
||||
'headers': {'Connection': 'Close'},
|
||||
'auth': aiohttp.BasicAuth(self._write_key, ''),
|
||||
'json': data,
|
||||
'cookies': self.cookies
|
||||
}
|
||||
try:
|
||||
async with aiohttp.request(**request_kwargs) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except Exception as e:
|
||||
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
|
||||
|
||||
async def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if self._enabled:
|
||||
log.debug('Sending track event: %s', event)
|
||||
await self._post('/track', event)
|
||||
|
||||
async def send_new_download_start(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_success(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
|
||||
async def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
|
||||
'message': str(e),
|
||||
})
|
||||
|
||||
def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
self.analytics_api.track({
|
||||
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
await self.track({
|
||||
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
|
||||
'event': NEW_DOWNLOAD_STAT,
|
||||
'properties': self._event_properties({
|
||||
@@ -70,90 +117,53 @@ class Manager:
|
||||
'timestamp': utils.isonow(),
|
||||
})
|
||||
|
||||
def send_upnp_setup_success_fail(self, success, status):
|
||||
self.analytics_api.track(
|
||||
async def send_upnp_setup_success_fail(self, success, status):
|
||||
await self.track(
|
||||
self._event(UPNP_SETUP, {
|
||||
'success': success,
|
||||
'status': status,
|
||||
})
|
||||
)
|
||||
|
||||
def send_server_startup(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP))
|
||||
async def send_server_startup(self):
|
||||
await self.track(self._event(SERVER_STARTUP))
|
||||
|
||||
def send_server_startup_success(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
async def send_server_startup_success(self):
|
||||
await self.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
|
||||
def send_server_startup_error(self, message):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
async def send_server_startup_error(self, message):
|
||||
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
def send_download_started(self, id_, name, claim_dict=None):
|
||||
self.analytics_api.track(
|
||||
async def send_download_started(self, id_, name, claim_dict=None):
|
||||
await self.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
async def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
async def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
def send_claim_action(self, action):
|
||||
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
async def send_claim_action(self, action):
|
||||
await self.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
def send_new_channel(self):
|
||||
self.analytics_api.track(self._event(NEW_CHANNEL))
|
||||
async def send_new_channel(self):
|
||||
await self.track(self._event(NEW_CHANNEL))
|
||||
|
||||
def send_credits_sent(self):
|
||||
self.analytics_api.track(self._event(CREDITS_SENT))
|
||||
async def send_credits_sent(self):
|
||||
await self.track(self._event(CREDITS_SENT))
|
||||
|
||||
def _send_heartbeat(self):
|
||||
self.analytics_api.track(self._event(HEARTBEAT))
|
||||
async def _send_heartbeat(self):
|
||||
await self.track(self._event(HEARTBEAT))
|
||||
|
||||
def _update_tracked_metrics(self):
|
||||
async def _update_tracked_metrics(self):
|
||||
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
# Setup / Shutdown
|
||||
|
||||
def start(self):
|
||||
if not self.is_started:
|
||||
for name, _, interval in self._get_looping_calls():
|
||||
self.looping_call_manager.start(name, interval)
|
||||
self.is_started = True
|
||||
|
||||
def shutdown(self):
|
||||
self.looping_call_manager.shutdown()
|
||||
|
||||
def register_repeating_metric(self, event_name, value_generator, frequency=300):
|
||||
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
|
||||
self.looping_call_manager.register_looping_call(event_name, lcall)
|
||||
lcall.start(frequency)
|
||||
|
||||
def _get_looping_calls(self):
|
||||
return [
|
||||
('send_heartbeat', self._send_heartbeat, 300),
|
||||
('update_tracked_metrics', self._update_tracked_metrics, 600),
|
||||
]
|
||||
|
||||
def _setup_looping_calls(self):
|
||||
call_manager = looping_call_manager.LoopingCallManager()
|
||||
for name, fn, _ in self._get_looping_calls():
|
||||
call_manager.register_looping_call(name, task.LoopingCall(fn))
|
||||
return call_manager
|
||||
|
||||
def _send_repeating_metric(self, event_name, value_generator):
|
||||
result = value_generator()
|
||||
self._if_deferred(result, self._send_repeating_metric_value, event_name)
|
||||
|
||||
def _send_repeating_metric_value(self, result, event_name):
|
||||
should_send, value = result
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(event_name, value))
|
||||
await self.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self._tracked_data[metric].append(value)
|
||||
@@ -238,65 +248,3 @@ class Manager:
|
||||
context['os']['desktop'] = platform['desktop']
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
|
||||
if isinstance(maybe_deferred, defer.Deferred):
|
||||
maybe_deferred.addCallback(callback, *args, **kwargs)
|
||||
else:
|
||||
callback(maybe_deferred, *args, **kwargs)
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self, cookies, url, write_key, enabled):
|
||||
self.cookies = cookies
|
||||
self.url = url
|
||||
self._write_key = write_key
|
||||
self._enabled = enabled
|
||||
|
||||
def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
# If you make a request, wait just the right amount of time,
|
||||
# then make another request, the requests module may opt to
|
||||
# reuse the connection, but by the time the server gets it the
|
||||
# timeout will have expired.
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
def update_cookies(response):
|
||||
self.cookies.update(response.cookies())
|
||||
return response
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
headers = {b"Connection": b"close"}
|
||||
d = treq.post(self.url + endpoint, auth=(self._write_key, ''), json=data,
|
||||
headers=headers, cookies=self.cookies)
|
||||
d.addCallback(update_cookies)
|
||||
return d
|
||||
|
||||
def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if not self._enabled:
|
||||
return defer.succeed('Analytics disabled')
|
||||
|
||||
def _log_error(failure, event):
|
||||
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
|
||||
|
||||
log.debug('Sending track event: %s', event)
|
||||
d = self._post('/track', event)
|
||||
d.addErrback(_log_error, event)
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
"""Initialize an instance using values from the configuration"""
|
||||
if enabled is None:
|
||||
enabled = conf.settings['share_usage_data']
|
||||
return cls(
|
||||
{},
|
||||
conf.settings['ANALYTICS_ENDPOINT'],
|
||||
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
|
||||
enabled,
|
||||
)
|
||||
Whitespace-only changes.
@@ -1,34 +0,0 @@
|
||||
import logging
|
||||
from zope.interface import implementer
|
||||
from twisted.cred import portal, checkers, credentials, error as cred_error
|
||||
from twisted.internet import defer
|
||||
from twisted.web import resource
|
||||
from lbrynet.extras.daemon.auth.keyring import Keyring
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@implementer(portal.IRealm)
|
||||
class HttpPasswordRealm:
|
||||
def __init__(self, resource):
|
||||
self.resource = resource
|
||||
|
||||
def requestAvatar(self, avatarId, mind, *interfaces):
|
||||
log.debug("Processing request for %s", avatarId)
|
||||
if resource.IResource in interfaces:
|
||||
return (resource.IResource, self.resource, lambda: None)
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
@implementer(checkers.ICredentialsChecker)
|
||||
class PasswordChecker:
|
||||
credentialInterfaces = (credentials.IUsernamePassword,)
|
||||
|
||||
def __init__(self, keyring: Keyring):
|
||||
self.api_key = keyring.api_key
|
||||
|
||||
def requestAvatarId(self, creds):
|
||||
if creds.checkPassword(self.api_key.secret.encode()) and creds.username == self.api_key.name.encode():
|
||||
return defer.succeed(creds.username)
|
||||
log.warning('Incorrect username or password')
|
||||
return defer.fail(cred_error.UnauthorizedLogin('Incorrect username or password'))
|
||||
@@ -1,130 +0,0 @@
|
||||
import json
|
||||
import aiohttp
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.daemon.auth.keyring import Keyring, APIKey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
USER_AGENT = "AuthServiceProxy/0.1"
|
||||
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
|
||||
TWISTED_SESSION = "TWISTED_SESSION"
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
HTTP_TIMEOUT = 30
|
||||
|
||||
|
||||
class JSONRPCException(Exception):
|
||||
def __init__(self, rpc_error):
|
||||
super().__init__()
|
||||
self.error = rpc_error
|
||||
|
||||
|
||||
class UnAuthAPIClient:
|
||||
def __init__(self, host, port, session):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.session = session
|
||||
|
||||
def __getattr__(self, method):
|
||||
async def f(*args, **kwargs):
|
||||
return await self.call(method, [args, kwargs])
|
||||
|
||||
return f
|
||||
|
||||
@classmethod
|
||||
async def from_url(cls, url):
|
||||
url_fragment = urlparse(url)
|
||||
host = url_fragment.hostname
|
||||
port = url_fragment.port
|
||||
connector = aiohttp.TCPConnector(
|
||||
ssl=None if not conf.settings['use_https'] else Keyring.load_from_disk().ssl_context
|
||||
)
|
||||
session = aiohttp.ClientSession(connector=connector)
|
||||
return cls(host, port, session)
|
||||
|
||||
async def call(self, method, params=None):
|
||||
message = {'method': method, 'params': params}
|
||||
async with self.session.get(conf.settings.get_api_connection_string(), json=message) as resp:
|
||||
return await resp.json()
|
||||
|
||||
|
||||
class AuthAPIClient:
|
||||
def __init__(self, key, session, cookies, url, login_url):
|
||||
self.session = session
|
||||
self.__api_key = key
|
||||
self.__login_url = login_url
|
||||
self.__id_count = 0
|
||||
self.__url = url
|
||||
self.__cookies = cookies
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name.startswith('__') and name.endswith('__'):
|
||||
raise AttributeError(name)
|
||||
|
||||
def f(*args, **kwargs):
|
||||
return self.call(name, [args, kwargs])
|
||||
|
||||
return f
|
||||
|
||||
async def call(self, method, params=None):
|
||||
params = params or {}
|
||||
self.__id_count += 1
|
||||
|
||||
pre_auth_post_data = {
|
||||
'version': '2',
|
||||
'method': method,
|
||||
'params': params,
|
||||
'id': self.__id_count
|
||||
}
|
||||
to_auth = json.dumps(pre_auth_post_data, sort_keys=True)
|
||||
auth_msg = self.__api_key.get_hmac(to_auth).decode()
|
||||
pre_auth_post_data.update({'hmac': auth_msg})
|
||||
post_data = json.dumps(pre_auth_post_data)
|
||||
|
||||
headers = {
|
||||
'Host': self.__url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'
|
||||
}
|
||||
|
||||
async with self.session.post(self.__login_url, data=post_data, headers=headers) as resp:
|
||||
if resp is None:
|
||||
raise JSONRPCException({'code': -342, 'message': 'missing HTTP response from server'})
|
||||
resp.raise_for_status()
|
||||
|
||||
next_secret = resp.headers.get(LBRY_SECRET, False)
|
||||
if next_secret:
|
||||
self.__api_key.secret = next_secret
|
||||
|
||||
return await resp.json()
|
||||
|
||||
@classmethod
|
||||
async def get_client(cls, key_name=None):
|
||||
api_key_name = key_name or "api"
|
||||
keyring = Keyring.load_from_disk()
|
||||
|
||||
api_key = keyring.api_key
|
||||
login_url = conf.settings.get_api_connection_string(api_key_name, api_key.secret)
|
||||
url = urlparse(login_url)
|
||||
|
||||
headers = {
|
||||
'Host': url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'
|
||||
}
|
||||
connector = aiohttp.TCPConnector(ssl=None if not conf.settings['use_https'] else keyring.ssl_context)
|
||||
session = aiohttp.ClientSession(connector=connector)
|
||||
|
||||
async with session.post(login_url, headers=headers) as r:
|
||||
cookies = r.cookies
|
||||
uid = cookies.get(TWISTED_SECURE_SESSION if conf.settings['use_https'] else TWISTED_SESSION).value
|
||||
api_key = APIKey.create(seed=uid.encode())
|
||||
return cls(api_key, session, cookies, url, login_url)
|
||||
|
||||
|
||||
class LBRYAPIClient:
|
||||
@staticmethod
|
||||
def get_client():
|
||||
return AuthAPIClient.get_client() if conf.settings['use_auth_http'] else \
|
||||
UnAuthAPIClient.from_url(conf.settings.get_api_connection_string())
|
||||
@@ -1,50 +0,0 @@
|
||||
import logging
|
||||
|
||||
from twisted.web import server, guard, resource
|
||||
from twisted.cred import portal
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.daemon.auth.auth import PasswordChecker, HttpPasswordRealm
|
||||
from lbrynet.extras.daemon.auth.keyring import Keyring
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HTTPJSONRPCFactory(server.Site):
|
||||
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
|
||||
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
|
||||
self.use_ssl = False
|
||||
|
||||
|
||||
class HTTPSJSONRPCFactory(server.Site):
|
||||
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
|
||||
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
|
||||
self.options = keyring.private_certificate.options()
|
||||
self.use_ssl = True
|
||||
|
||||
|
||||
class AuthJSONRPCResource(resource.Resource):
|
||||
def __init__(self, protocol):
|
||||
resource.Resource.__init__(self)
|
||||
self.putChild(b"", protocol)
|
||||
self.putChild(conf.settings['API_ADDRESS'].encode(), protocol)
|
||||
|
||||
def getChild(self, name, request):
|
||||
request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
|
||||
request.setHeader('expires', '0')
|
||||
return self if name == '' else resource.Resource.getChild(self, name, request)
|
||||
|
||||
def getServerFactory(self, keyring: Keyring, use_authentication: bool, use_https: bool) -> server.Site:
|
||||
factory_class = HTTPSJSONRPCFactory if use_https else HTTPJSONRPCFactory
|
||||
if use_authentication:
|
||||
log.info("Using authenticated API")
|
||||
checker = PasswordChecker(keyring)
|
||||
realm = HttpPasswordRealm(self)
|
||||
portal_to_realm = portal.Portal(realm, [checker, ])
|
||||
root = guard.HTTPAuthSessionWrapper(
|
||||
portal_to_realm, [guard.BasicCredentialFactory(b'Login to lbrynet api'), ]
|
||||
)
|
||||
else:
|
||||
log.info("Using non-authenticated API")
|
||||
root = self
|
||||
return factory_class(root, keyring)
|
||||
@@ -1,131 +0,0 @@
|
||||
import os
|
||||
import datetime
|
||||
import hmac
|
||||
import hashlib
|
||||
import base58
|
||||
import logging
|
||||
from OpenSSL.crypto import FILETYPE_PEM
|
||||
from ssl import create_default_context, SSLContext
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.x509.name import NameOID, NameAttribute
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from twisted.internet import ssl
|
||||
|
||||
from lbrynet import conf
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def sha(x: bytes) -> str:
|
||||
h = hashlib.sha256(x).digest()
|
||||
return base58.b58encode(h).decode()
|
||||
|
||||
|
||||
def generate_key(x: bytes = None) -> str:
|
||||
if not x:
|
||||
return sha(os.urandom(256))
|
||||
else:
|
||||
return sha(x)
|
||||
|
||||
|
||||
class APIKey:
|
||||
def __init__(self, secret: str, name: str):
|
||||
self.secret = secret
|
||||
self.name = name
|
||||
|
||||
@classmethod
|
||||
def create(cls, seed=None, name=None):
|
||||
secret = generate_key(seed)
|
||||
return APIKey(secret, name)
|
||||
|
||||
def _raw_key(self) -> str:
|
||||
return base58.b58decode(self.secret)
|
||||
|
||||
def get_hmac(self, message) -> str:
|
||||
decoded_key = self._raw_key()
|
||||
signature = hmac.new(decoded_key, message.encode(), hashlib.sha256)
|
||||
return base58.b58encode(signature.digest())
|
||||
|
||||
def compare_hmac(self, message, token) -> bool:
|
||||
decoded_token = base58.b58decode(token)
|
||||
target = base58.b58decode(self.get_hmac(message))
|
||||
|
||||
try:
|
||||
if len(decoded_token) != len(target):
|
||||
return False
|
||||
return hmac.compare_digest(decoded_token, target)
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
class Keyring:
|
||||
encoding = serialization.Encoding.PEM
|
||||
filetype = FILETYPE_PEM
|
||||
|
||||
def __init__(self, api_key: APIKey, public_certificate: str, private_certificate: ssl.PrivateCertificate = None):
|
||||
self.api_key: APIKey = api_key
|
||||
self.public_certificate: str = public_certificate
|
||||
self.private_certificate: (ssl.PrivateCertificate or None) = private_certificate
|
||||
self.ssl_context: SSLContext = create_default_context(cadata=self.public_certificate)
|
||||
|
||||
@classmethod
|
||||
def load_from_disk(cls):
|
||||
api_key_path = os.path.join(conf.settings.data_dir, 'auth_token')
|
||||
api_ssl_cert_path = os.path.join(conf.settings.data_dir, 'api_ssl_cert.pem')
|
||||
if not os.path.isfile(api_key_path) or not os.path.isfile(api_ssl_cert_path):
|
||||
return
|
||||
with open(api_key_path, 'rb') as f:
|
||||
api_key = APIKey(f.read().decode(), "api")
|
||||
with open(api_ssl_cert_path, 'rb') as f:
|
||||
public_cert = f.read().decode()
|
||||
return cls(api_key, public_cert)
|
||||
|
||||
@classmethod
|
||||
def generate_and_save(cls):
|
||||
dns = conf.settings['api_host']
|
||||
private_key = rsa.generate_private_key(
|
||||
public_exponent=65537,
|
||||
key_size=4096,
|
||||
backend=default_backend()
|
||||
)
|
||||
subject = issuer = x509.Name([
|
||||
NameAttribute(NameOID.COUNTRY_NAME, "US"),
|
||||
NameAttribute(NameOID.ORGANIZATION_NAME, "LBRY"),
|
||||
NameAttribute(NameOID.COMMON_NAME, "LBRY API"),
|
||||
])
|
||||
alternative_name = x509.SubjectAlternativeName([x509.DNSName(dns)])
|
||||
certificate = x509.CertificateBuilder(
|
||||
subject_name=subject,
|
||||
issuer_name=issuer,
|
||||
public_key=private_key.public_key(),
|
||||
serial_number=x509.random_serial_number(),
|
||||
not_valid_before=datetime.datetime.utcnow(),
|
||||
not_valid_after=datetime.datetime.utcnow() + datetime.timedelta(days=365),
|
||||
extensions=[x509.Extension(oid=alternative_name.oid, critical=False, value=alternative_name)]
|
||||
).sign(private_key, hashes.SHA256(), default_backend())
|
||||
public_certificate = certificate.public_bytes(cls.encoding).decode()
|
||||
private_certificate = ssl.PrivateCertificate.load(
|
||||
public_certificate,
|
||||
ssl.KeyPair.load(
|
||||
private_key.private_bytes(
|
||||
encoding=cls.encoding,
|
||||
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
encryption_algorithm=serialization.NoEncryption()
|
||||
).decode(),
|
||||
cls.filetype
|
||||
),
|
||||
cls.filetype
|
||||
)
|
||||
|
||||
auth_token = APIKey.create(seed=None, name="api")
|
||||
|
||||
with open(os.path.join(conf.settings.data_dir, 'auth_token'), 'wb') as f:
|
||||
f.write(auth_token.secret.encode())
|
||||
|
||||
with open(os.path.join(conf.settings.data_dir, 'api_ssl_cert.pem'), 'wb') as f:
|
||||
f.write(public_certificate.encode())
|
||||
|
||||
return cls(auth_token, public_certificate, private_certificate)
|
||||
@@ -1,611 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from urllib import parse as urlparse
|
||||
import json
|
||||
import inspect
|
||||
import signal
|
||||
from functools import wraps
|
||||
from twisted.web import server
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet.error import ConnectionDone, ConnectionLost
|
||||
from txjsonrpc import jsonrpclib
|
||||
from traceback import format_exc
|
||||
|
||||
from lbrynet import conf, utils
|
||||
from lbrynet.extras.daemon import analytics
|
||||
from lbrynet.p2p.Error import InvalidAuthenticationToken
|
||||
from lbrynet.p2p.Error import ComponentsNotStarted, ComponentStartConditionNotMet
|
||||
from lbrynet.extras.looping_call_manager import LoopingCallManager
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.auth.keyring import APIKey, Keyring
|
||||
from lbrynet.extras.daemon.auth.undecorated import undecorated
|
||||
from lbrynet.extras.daemon.auth.factory import AuthJSONRPCResource
|
||||
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
EMPTY_PARAMS = [{}]
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
|
||||
|
||||
class JSONRPCError:
|
||||
# http://www.jsonrpc.org/specification#error_object
|
||||
CODE_PARSE_ERROR = -32700 # Invalid JSON. Error while parsing the JSON text.
|
||||
CODE_INVALID_REQUEST = -32600 # The JSON sent is not a valid Request object.
|
||||
CODE_METHOD_NOT_FOUND = -32601 # The method does not exist / is not available.
|
||||
CODE_INVALID_PARAMS = -32602 # Invalid method parameter(s).
|
||||
CODE_INTERNAL_ERROR = -32603 # Internal JSON-RPC error (I think this is like a 500?)
|
||||
CODE_APPLICATION_ERROR = -32500 # Generic error with our app??
|
||||
CODE_AUTHENTICATION_ERROR = -32501 # Authentication failed
|
||||
|
||||
MESSAGES = {
|
||||
CODE_PARSE_ERROR: "Parse Error. Data is not valid JSON.",
|
||||
CODE_INVALID_REQUEST: "JSON data is not a valid Request",
|
||||
CODE_METHOD_NOT_FOUND: "Method Not Found",
|
||||
CODE_INVALID_PARAMS: "Invalid Params",
|
||||
CODE_INTERNAL_ERROR: "Internal Error",
|
||||
CODE_AUTHENTICATION_ERROR: "Authentication Failed",
|
||||
}
|
||||
|
||||
HTTP_CODES = {
|
||||
CODE_INVALID_REQUEST: 400,
|
||||
CODE_PARSE_ERROR: 400,
|
||||
CODE_INVALID_PARAMS: 400,
|
||||
CODE_METHOD_NOT_FOUND: 404,
|
||||
CODE_INTERNAL_ERROR: 500,
|
||||
CODE_APPLICATION_ERROR: 500,
|
||||
CODE_AUTHENTICATION_ERROR: 401,
|
||||
}
|
||||
|
||||
def __init__(self, message, code=CODE_APPLICATION_ERROR, traceback=None, data=None):
|
||||
assert isinstance(code, int), "'code' must be an int"
|
||||
assert (data is None or isinstance(data, dict)), "'data' must be None or a dict"
|
||||
self.code = code
|
||||
if message is None:
|
||||
message = self.MESSAGES[code] if code in self.MESSAGES else "API Error"
|
||||
self.message = message
|
||||
self.data = {} if data is None else data
|
||||
self.traceback = []
|
||||
if traceback is not None:
|
||||
trace_lines = traceback.split("\n")
|
||||
for i, t in enumerate(trace_lines):
|
||||
if "--- <exception caught here> ---" in t:
|
||||
if len(trace_lines) > i + 1:
|
||||
self.traceback = [j for j in trace_lines[i+1:] if j]
|
||||
break
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'code': self.code,
|
||||
'message': self.message,
|
||||
'data': self.traceback
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def create_from_exception(cls, message, code=CODE_APPLICATION_ERROR, traceback=None):
|
||||
return cls(message, code=code, traceback=traceback)
|
||||
|
||||
|
||||
class UnknownAPIMethodError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def trap(err, *to_trap):
|
||||
err.trap(*to_trap)
|
||||
|
||||
|
||||
def jsonrpc_dumps_pretty(obj, **kwargs):
|
||||
try:
|
||||
id_ = kwargs.pop("id")
|
||||
except KeyError:
|
||||
id_ = None
|
||||
|
||||
if isinstance(obj, JSONRPCError):
|
||||
data = {"jsonrpc": "2.0", "error": obj.to_dict(), "id": id_}
|
||||
else:
|
||||
data = {"jsonrpc": "2.0", "result": obj, "id": id_}
|
||||
|
||||
return json.dumps(data, cls=JSONResponseEncoder, sort_keys=True, indent=2, **kwargs) + "\n"
|
||||
|
||||
|
||||
class JSONRPCServerType(type):
|
||||
def __new__(mcs, name, bases, newattrs):
|
||||
klass = type.__new__(mcs, name, bases, newattrs)
|
||||
klass.callable_methods = {}
|
||||
klass.deprecated_methods = {}
|
||||
|
||||
for methodname in dir(klass):
|
||||
if methodname.startswith("jsonrpc_"):
|
||||
method = getattr(klass, methodname)
|
||||
if not hasattr(method, '_deprecated'):
|
||||
klass.callable_methods.update({methodname.split("jsonrpc_")[1]: method})
|
||||
else:
|
||||
klass.deprecated_methods.update({methodname.split("jsonrpc_")[1]: method})
|
||||
return klass
|
||||
|
||||
|
||||
class AuthorizedBase(metaclass=JSONRPCServerType):
|
||||
|
||||
@staticmethod
|
||||
def deprecated(new_command=None):
|
||||
def _deprecated_wrapper(f):
|
||||
f.new_command = new_command
|
||||
f._deprecated = True
|
||||
return f
|
||||
return _deprecated_wrapper
|
||||
|
||||
@staticmethod
|
||||
def requires(*components, **conditions):
|
||||
if conditions and ["conditions"] != list(conditions.keys()):
|
||||
raise SyntaxError("invalid conditions argument")
|
||||
condition_names = conditions.get("conditions", [])
|
||||
|
||||
def _wrap(fn):
|
||||
@wraps(fn)
|
||||
def _inner(*args, **kwargs):
|
||||
component_manager = args[0].component_manager
|
||||
for condition_name in condition_names:
|
||||
condition_result, err_msg = component_manager.evaluate_condition(condition_name)
|
||||
if not condition_result:
|
||||
raise ComponentStartConditionNotMet(err_msg)
|
||||
if not component_manager.all_components_running(*components):
|
||||
raise ComponentsNotStarted("the following required components have not yet started: "
|
||||
"%s" % json.dumps(components))
|
||||
return fn(*args, **kwargs)
|
||||
return _inner
|
||||
return _wrap
|
||||
|
||||
|
||||
class AuthJSONRPCServer(AuthorizedBase):
|
||||
"""
|
||||
Authorized JSONRPC server used as the base class for the LBRY API
|
||||
|
||||
API methods are named with a leading "jsonrpc_"
|
||||
|
||||
Attributes:
|
||||
sessions (dict): (dict): {<session id>: <lbrynet.daemon.auth.util.APIKey>}
|
||||
callable_methods (dict): {<api method name>: <api method>}
|
||||
|
||||
Authentication:
|
||||
If use_authentication is true, basic HTTP and HMAC authentication will be used for all requests and the
|
||||
service url will require a username and password.
|
||||
|
||||
To start an authenticated session a client sends an HTTP POST to <user>:<password>@<api host>:<api port>.
|
||||
If accepted, the server replies with a TWISTED_SESSION cookie containing a session id and the message "OK".
|
||||
The client initializes their shared secret for hmac to be the b64 encoded sha256 of their session id.
|
||||
|
||||
To send an authenticated request a client sends an HTTP POST to the auth api url with the TWISTED_SESSION
|
||||
cookie and includes a hmac token in the message using the previously set shared secret. If the token is valid
|
||||
the server will randomize the shared secret and return the new value under the LBRY_SECRET header, which the
|
||||
client uses to generate the token for their next request.
|
||||
"""
|
||||
#implements(resource.IResource)
|
||||
|
||||
isLeaf = True
|
||||
allowed_during_startup = []
|
||||
component_attributes = {}
|
||||
|
||||
def __init__(self, analytics_manager=None, component_manager=None, use_authentication=None, use_https=None,
|
||||
to_skip=None, looping_calls=None, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.analytics_manager = analytics_manager or analytics.Manager.new_instance()
|
||||
self.component_manager = component_manager or ComponentManager(
|
||||
analytics_manager=self.analytics_manager,
|
||||
skip_components=to_skip or [],
|
||||
reactor=reactor
|
||||
)
|
||||
self.looping_call_manager = LoopingCallManager({n: lc for n, (lc, t) in (looping_calls or {}).items()})
|
||||
self._looping_call_times = {n: t for n, (lc, t) in (looping_calls or {}).items()}
|
||||
self._use_authentication = use_authentication or conf.settings['use_auth_http']
|
||||
self._use_https = use_https or conf.settings['use_https']
|
||||
self.listening_port = None
|
||||
self._component_setup_deferred = None
|
||||
self.announced_startup = False
|
||||
self.sessions = {}
|
||||
self.server = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_listening(self):
|
||||
from twisted.internet import reactor, error as tx_error
|
||||
|
||||
try:
|
||||
self.server = self.get_server_factory()
|
||||
if self.server.use_ssl:
|
||||
log.info("Using SSL")
|
||||
self.listening_port = reactor.listenSSL(
|
||||
conf.settings['api_port'], self.server, self.server.options, interface=conf.settings['api_host']
|
||||
)
|
||||
else:
|
||||
log.info("Not using SSL")
|
||||
self.listening_port = reactor.listenTCP(
|
||||
conf.settings['api_port'], self.server, interface=conf.settings['api_host']
|
||||
)
|
||||
log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
|
||||
yield self.setup()
|
||||
self.analytics_manager.send_server_startup_success()
|
||||
except tx_error.CannotListenError:
|
||||
log.error('lbrynet API failed to bind TCP %s:%i for listening. Daemon is already running or this port is '
|
||||
'already in use by another application.', conf.settings['api_host'], conf.settings['api_port'])
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
except defer.CancelledError:
|
||||
log.info("shutting down before finished starting")
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
except Exception as err:
|
||||
self.analytics_manager.send_server_startup_error(str(err))
|
||||
log.exception('Failed to start lbrynet-daemon')
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
|
||||
def setup(self):
|
||||
from twisted.internet import reactor
|
||||
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
|
||||
if not self.analytics_manager.is_started:
|
||||
self.analytics_manager.start()
|
||||
self.analytics_manager.send_server_startup()
|
||||
for lc_name, lc_time in self._looping_call_times.items():
|
||||
self.looping_call_manager.start(lc_name, lc_time)
|
||||
|
||||
def update_attribute(setup_result, component):
|
||||
setattr(self, self.component_attributes[component.component_name], component.component)
|
||||
|
||||
kwargs = {component: update_attribute for component in self.component_attributes.keys()}
|
||||
self._component_setup_deferred = self.component_manager.setup(**kwargs)
|
||||
return self._component_setup_deferred
|
||||
|
||||
@staticmethod
|
||||
def _already_shutting_down(sig_num, frame):
|
||||
log.info("Already shutting down")
|
||||
|
||||
def _shutdown(self):
|
||||
# ignore INT/TERM signals once shutdown has started
|
||||
signal.signal(signal.SIGINT, self._already_shutting_down)
|
||||
signal.signal(signal.SIGTERM, self._already_shutting_down)
|
||||
if self.listening_port:
|
||||
self.listening_port.stopListening()
|
||||
self.looping_call_manager.shutdown()
|
||||
if self.server is not None:
|
||||
for session in list(self.server.sessions.values()):
|
||||
session.expire()
|
||||
if self.analytics_manager:
|
||||
self.analytics_manager.shutdown()
|
||||
try:
|
||||
self._component_setup_deferred.cancel()
|
||||
except (AttributeError, defer.CancelledError):
|
||||
pass
|
||||
if self.component_manager is not None:
|
||||
d = self.component_manager.stop()
|
||||
d.addErrback(log.fail(), 'Failure while shutting down')
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
return d
|
||||
|
||||
def get_server_factory(self):
|
||||
keyring = None
|
||||
if self._use_authentication or self._use_https:
|
||||
keyring = Keyring.generate_and_save()
|
||||
return AuthJSONRPCResource(self).getServerFactory(keyring, self._use_authentication, self._use_https)
|
||||
|
||||
def _set_headers(self, request, data, update_secret=False):
|
||||
if conf.settings['allowed_origin']:
|
||||
request.setHeader("Access-Control-Allow-Origin", conf.settings['allowed_origin'])
|
||||
request.setHeader("Content-Type", "application/json")
|
||||
request.setHeader("Accept", "application/json-rpc")
|
||||
request.setHeader("Content-Length", str(len(data)))
|
||||
if update_secret:
|
||||
session_id = request.getSession().uid
|
||||
request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret)
|
||||
|
||||
@staticmethod
|
||||
def _render_message(request, message: str):
|
||||
request.write(message.encode())
|
||||
request.finish()
|
||||
|
||||
def _render_error(self, failure, request, id_):
|
||||
if isinstance(failure, JSONRPCError):
|
||||
error = failure
|
||||
elif isinstance(failure, Failure):
|
||||
# maybe failure is JSONRPCError wrapped in a twisted Failure
|
||||
error = failure.check(JSONRPCError)
|
||||
if error is None:
|
||||
# maybe its a twisted Failure with another type of error
|
||||
if hasattr(failure.type, "code"):
|
||||
error_code = failure.type.code
|
||||
else:
|
||||
error_code = JSONRPCError.CODE_APPLICATION_ERROR
|
||||
error = JSONRPCError.create_from_exception(
|
||||
failure.getErrorMessage() or failure.type.__name__,
|
||||
code=error_code,
|
||||
traceback=failure.getTraceback()
|
||||
)
|
||||
if not failure.check(ComponentsNotStarted, ComponentStartConditionNotMet):
|
||||
log.warning("error processing api request: %s\ntraceback: %s", error.message,
|
||||
"\n".join(error.traceback))
|
||||
else:
|
||||
# last resort, just cast it as a string
|
||||
error = JSONRPCError(str(failure))
|
||||
|
||||
response_content = jsonrpc_dumps_pretty(error, id=id_, ledger=self.ledger)
|
||||
self._set_headers(request, response_content)
|
||||
request.setResponseCode(200)
|
||||
self._render_message(request, response_content)
|
||||
|
||||
@staticmethod
|
||||
def _handle_dropped_request(result, d, function_name):
|
||||
if not d.called:
|
||||
log.warning("Cancelling dropped api request %s", function_name)
|
||||
d.cancel()
|
||||
|
||||
def render(self, request):
|
||||
try:
|
||||
return self._render(request)
|
||||
except BaseException as e:
|
||||
log.error(e)
|
||||
error = JSONRPCError.create_from_exception(str(e), traceback=format_exc())
|
||||
self._render_error(error, request, None)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def _render(self, request):
|
||||
time_in = utils.now()
|
||||
# if not self._check_headers(request):
|
||||
# self._render_error(Failure(InvalidHeaderError()), request, None)
|
||||
# return server.NOT_DONE_YET
|
||||
session = request.getSession()
|
||||
session_id = session.uid
|
||||
finished_deferred = request.notifyFinish()
|
||||
|
||||
if self._use_authentication:
|
||||
# if this is a new session, send a new secret and set the expiration
|
||||
# otherwise, session.touch()
|
||||
if self._initialize_session(session_id):
|
||||
def expire_session():
|
||||
self._unregister_user_session(session_id)
|
||||
|
||||
session.notifyOnExpire(expire_session)
|
||||
message = "OK"
|
||||
request.setResponseCode(200)
|
||||
self._set_headers(request, message, True)
|
||||
self._render_message(request, message)
|
||||
return server.NOT_DONE_YET
|
||||
else:
|
||||
session.touch()
|
||||
|
||||
request.content.seek(0, 0)
|
||||
content = request.content.read().decode()
|
||||
try:
|
||||
parsed = jsonrpclib.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
log.warning("Unable to decode request json")
|
||||
self._render_error(JSONRPCError(None, code=JSONRPCError.CODE_PARSE_ERROR), request, None)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
request_id = None
|
||||
try:
|
||||
function_name = parsed.get('method')
|
||||
args = parsed.get('params', {})
|
||||
request_id = parsed.get('id', None)
|
||||
token = parsed.pop('hmac', None)
|
||||
except AttributeError as err:
|
||||
log.warning(err)
|
||||
self._render_error(
|
||||
JSONRPCError(None, code=JSONRPCError.CODE_INVALID_REQUEST), request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
reply_with_next_secret = False
|
||||
if self._use_authentication:
|
||||
try:
|
||||
self._verify_token(session_id, parsed, token)
|
||||
except InvalidAuthenticationToken as err:
|
||||
log.warning("API validation failed")
|
||||
self._render_error(
|
||||
JSONRPCError.create_from_exception(
|
||||
str(err),
|
||||
code=JSONRPCError.CODE_AUTHENTICATION_ERROR,
|
||||
traceback=format_exc()
|
||||
),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
request.addCookie("TWISTED_SESSION", session_id)
|
||||
self._update_session_secret(session_id)
|
||||
reply_with_next_secret = True
|
||||
|
||||
try:
|
||||
fn = self._get_jsonrpc_method(function_name)
|
||||
except UnknownAPIMethodError as err:
|
||||
log.warning('Failed to get function %s: %s', function_name, err)
|
||||
self._render_error(
|
||||
JSONRPCError(None, code=JSONRPCError.CODE_METHOD_NOT_FOUND),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
if args in (EMPTY_PARAMS, []):
|
||||
_args, _kwargs = (), {}
|
||||
elif isinstance(args, dict):
|
||||
_args, _kwargs = (), args
|
||||
elif len(args) == 1 and isinstance(args[0], dict):
|
||||
# TODO: this is for backwards compatibility. Remove this once API and UI are updated
|
||||
# TODO: also delete EMPTY_PARAMS then
|
||||
_args, _kwargs = (), args[0]
|
||||
elif len(args) == 2 and isinstance(args[0], list) and isinstance(args[1], dict):
|
||||
_args, _kwargs = args
|
||||
else:
|
||||
raise ValueError('invalid args format')
|
||||
|
||||
params_error, erroneous_params = self._check_params(fn, _args, _kwargs)
|
||||
if params_error is not None:
|
||||
params_error_message = '{} for {} command: {}'.format(
|
||||
params_error, function_name, ', '.join(erroneous_params)
|
||||
)
|
||||
log.warning(params_error_message)
|
||||
self._render_error(
|
||||
JSONRPCError(params_error_message, code=JSONRPCError.CODE_INVALID_PARAMS),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
try:
|
||||
result = fn(self, *_args, **_kwargs)
|
||||
if isinstance(result, Deferred):
|
||||
d = result
|
||||
elif isinstance(result, Failure):
|
||||
d = defer.fail(result)
|
||||
elif asyncio.iscoroutine(result):
|
||||
d = Deferred.fromFuture(asyncio.ensure_future(result))
|
||||
else:
|
||||
d = defer.succeed(result)
|
||||
except:
|
||||
d = defer.fail(Failure(captureVars=Deferred.debug))
|
||||
|
||||
# finished_deferred will callback when the request is finished
|
||||
# and errback if something went wrong. If the errback is
|
||||
# called, cancel the deferred stack. This is to prevent
|
||||
# request.finish() from being called on a closed request.
|
||||
finished_deferred.addErrback(self._handle_dropped_request, d, function_name)
|
||||
|
||||
d.addCallback(self._callback_render, request, request_id, reply_with_next_secret)
|
||||
d.addErrback(trap, ConnectionDone, ConnectionLost, defer.CancelledError)
|
||||
d.addErrback(self._render_error, request, request_id)
|
||||
d.addBoth(lambda _: log.debug("%s took %f",
|
||||
function_name,
|
||||
(utils.now() - time_in).total_seconds()))
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def _register_user_session(self, session_id):
|
||||
"""
|
||||
Add or update a HMAC secret for a session
|
||||
|
||||
@param session_id:
|
||||
@return: secret
|
||||
"""
|
||||
log.info("Started new api session")
|
||||
token = APIKey.create(seed=session_id)
|
||||
self.sessions.update({session_id: token})
|
||||
|
||||
def _unregister_user_session(self, session_id):
|
||||
log.info("Unregister API session")
|
||||
del self.sessions[session_id]
|
||||
|
||||
def _check_headers(self, request):
|
||||
return (
|
||||
self._check_header_source(request, 'Origin') and
|
||||
self._check_header_source(request, 'Referer')
|
||||
)
|
||||
|
||||
def _check_header_source(self, request, header):
|
||||
"""Check if the source of the request is allowed based on the header value."""
|
||||
source = request.getHeader(header)
|
||||
if not self._check_source_of_request(source):
|
||||
log.warning("Attempted api call from invalid %s: %s", header, source)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _check_source_of_request(self, source):
|
||||
if source is None:
|
||||
return True
|
||||
if conf.settings['api_host'] == '0.0.0.0':
|
||||
return True
|
||||
server, port = self.get_server_port(source)
|
||||
return self._check_server_port(server, port)
|
||||
|
||||
def _check_server_port(self, server, port):
|
||||
api = (conf.settings['api_host'], conf.settings['api_port'])
|
||||
return (server, port) == api or self._is_from_allowed_origin(server, port)
|
||||
|
||||
def _is_from_allowed_origin(self, server, port):
|
||||
allowed_origin = conf.settings['allowed_origin']
|
||||
if not allowed_origin:
|
||||
return False
|
||||
if allowed_origin == '*':
|
||||
return True
|
||||
allowed_server, allowed_port = self.get_server_port(allowed_origin)
|
||||
return (allowed_server, allowed_port) == (server, port)
|
||||
|
||||
def get_server_port(self, origin):
|
||||
parsed = urlparse.urlparse(origin)
|
||||
server_port = parsed.netloc.split(':')
|
||||
assert len(server_port) <= 2
|
||||
if len(server_port) == 2:
|
||||
return server_port[0], int(server_port[1])
|
||||
else:
|
||||
return server_port[0], 80
|
||||
|
||||
def _verify_method_is_callable(self, function_path):
|
||||
if function_path not in self.callable_methods:
|
||||
raise UnknownAPIMethodError(function_path)
|
||||
|
||||
def _get_jsonrpc_method(self, function_path):
|
||||
if function_path in self.deprecated_methods:
|
||||
new_command = self.deprecated_methods[function_path].new_command
|
||||
log.warning('API function \"%s\" is deprecated, please update to use \"%s\"',
|
||||
function_path, new_command)
|
||||
function_path = new_command
|
||||
self._verify_method_is_callable(function_path)
|
||||
return self.callable_methods.get(function_path)
|
||||
|
||||
@staticmethod
|
||||
def _check_params(function, args_tup, args_dict):
|
||||
argspec = inspect.getfullargspec(undecorated(function))
|
||||
num_optional_params = 0 if argspec.defaults is None else len(argspec.defaults)
|
||||
|
||||
duplicate_params = [
|
||||
duplicate_param
|
||||
for duplicate_param in argspec.args[1:len(args_tup) + 1]
|
||||
if duplicate_param in args_dict
|
||||
]
|
||||
|
||||
if duplicate_params:
|
||||
return 'Duplicate parameters', duplicate_params
|
||||
|
||||
missing_required_params = [
|
||||
required_param
|
||||
for required_param in argspec.args[len(args_tup)+1:-num_optional_params]
|
||||
if required_param not in args_dict
|
||||
]
|
||||
if len(missing_required_params):
|
||||
return 'Missing required parameters', missing_required_params
|
||||
|
||||
extraneous_params = [] if argspec.varkw is not None else [
|
||||
extra_param
|
||||
for extra_param in args_dict
|
||||
if extra_param not in argspec.args[1:]
|
||||
]
|
||||
if len(extraneous_params):
|
||||
return 'Extraneous parameters', extraneous_params
|
||||
|
||||
return None, None
|
||||
|
||||
def _initialize_session(self, session_id):
|
||||
if not self.sessions.get(session_id):
|
||||
self._register_user_session(session_id)
|
||||
return True
|
||||
return False
|
||||
|
||||
def _verify_token(self, session_id, message, token):
|
||||
if token is None:
|
||||
raise InvalidAuthenticationToken('Authentication token not found')
|
||||
to_auth = json.dumps(message, sort_keys=True)
|
||||
api_key = self.sessions.get(session_id)
|
||||
if not api_key.compare_hmac(to_auth, token):
|
||||
raise InvalidAuthenticationToken('Invalid authentication token')
|
||||
|
||||
def _update_session_secret(self, session_id):
|
||||
self.sessions.update({session_id: APIKey.create(name=session_id)})
|
||||
|
||||
def _callback_render(self, result, request, id_, auth_required=False):
|
||||
try:
|
||||
message = jsonrpc_dumps_pretty(result, id=id_, ledger=self.ledger)
|
||||
request.setResponseCode(200)
|
||||
self._set_headers(request, message, auth_required)
|
||||
self._render_message(request, message)
|
||||
except Exception as err:
|
||||
log.exception("Failed to render API response: %s", result)
|
||||
self._render_error(err, request, id_)
|
||||
|
||||
@staticmethod
|
||||
def _render_response(result):
|
||||
return defer.succeed(result)
|
||||
+27
-28
@@ -1,11 +1,9 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import time
|
||||
import logging
|
||||
import json
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.task import LoopingCall
|
||||
|
||||
from lbrynet.p2p.Error import InvalidExchangeRateResponse, CurrencyConversionError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -38,14 +36,14 @@ class MarketFeed:
|
||||
REQUESTS_TIMEOUT = 20
|
||||
EXCHANGE_RATE_UPDATE_RATE_SEC = 300
|
||||
|
||||
def __init__(self, market, name, url, params, fee):
|
||||
def __init__(self, market: str, name: str, url: str, params, fee):
|
||||
self.market = market
|
||||
self.name = name
|
||||
self.url = url
|
||||
self.params = params
|
||||
self.fee = fee
|
||||
self.rate = None
|
||||
self._updater = LoopingCall(self._update_price)
|
||||
self._task: asyncio.Task = None
|
||||
self._online = True
|
||||
|
||||
def rate_is_initialized(self):
|
||||
@@ -54,17 +52,16 @@ class MarketFeed:
|
||||
def is_online(self):
|
||||
return self._online
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _make_request(self):
|
||||
response = yield treq.get(self.url, params=self.params, timeout=self.REQUESTS_TIMEOUT)
|
||||
defer.returnValue((yield response.content()))
|
||||
async def _make_request(self):
|
||||
async with aiohttp.request('get', self.url, params=self.params) as response:
|
||||
return (await response.read()).decode()
|
||||
|
||||
def _handle_response(self, response):
|
||||
return NotImplementedError
|
||||
raise NotImplementedError()
|
||||
|
||||
def _subtract_fee(self, from_amount):
|
||||
# increase amount to account for market fees
|
||||
return defer.succeed(from_amount / (1.0 - self.fee))
|
||||
return from_amount / (1.0 - self.fee)
|
||||
|
||||
def _save_price(self, price):
|
||||
log.debug("Saving price update %f for %s from %s" % (price, self.market, self.name))
|
||||
@@ -77,21 +74,23 @@ class MarketFeed:
|
||||
log.debug("Exchange rate error (%s from %s): %s", self.market, self.name, err)
|
||||
self._online = False
|
||||
|
||||
def _update_price(self):
|
||||
d = self._make_request()
|
||||
d.addCallback(self._handle_response)
|
||||
d.addCallback(self._subtract_fee)
|
||||
d.addCallback(self._save_price)
|
||||
d.addErrback(self._on_error)
|
||||
return d
|
||||
async def _update_price(self):
|
||||
while True:
|
||||
try:
|
||||
response = await asyncio.wait_for(self._make_request(), self.REQUESTS_TIMEOUT)
|
||||
self._save_price(self._subtract_fee(self._handle_response(response)))
|
||||
except (asyncio.TimeoutError, InvalidExchangeRateResponse) as err:
|
||||
self._on_error(err)
|
||||
await asyncio.sleep(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
|
||||
|
||||
def start(self):
|
||||
if not self._updater.running:
|
||||
self._updater.start(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
|
||||
if not self._task:
|
||||
self._task = asyncio.create_task(self._update_price())
|
||||
|
||||
def stop(self):
|
||||
if self._updater.running:
|
||||
self._updater.stop()
|
||||
if self._task and not self._task.done():
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
|
||||
|
||||
class BittrexFeed(MarketFeed):
|
||||
@@ -116,7 +115,7 @@ class BittrexFeed(MarketFeed):
|
||||
if totals <= 0 or qtys <= 0:
|
||||
raise InvalidExchangeRateResponse(self.market, 'quantities were not positive')
|
||||
vwap = totals / qtys
|
||||
return defer.succeed(float(1.0 / vwap))
|
||||
return float(1.0 / vwap)
|
||||
|
||||
|
||||
class LBRYioFeed(MarketFeed):
|
||||
@@ -133,7 +132,7 @@ class LBRYioFeed(MarketFeed):
|
||||
json_response = json.loads(response)
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(1.0 / json_response['data']['lbc_btc'])
|
||||
return 1.0 / json_response['data']['lbc_btc']
|
||||
|
||||
|
||||
class LBRYioBTCFeed(MarketFeed):
|
||||
@@ -153,7 +152,7 @@ class LBRYioBTCFeed(MarketFeed):
|
||||
raise InvalidExchangeRateResponse(self.name, "invalid rate response : %s" % response)
|
||||
if 'data' not in json_response:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(1.0 / json_response['data']['btc_usd'])
|
||||
return 1.0 / json_response['data']['btc_usd']
|
||||
|
||||
|
||||
class CryptonatorBTCFeed(MarketFeed):
|
||||
@@ -174,7 +173,7 @@ class CryptonatorBTCFeed(MarketFeed):
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(float(json_response['ticker']['price']))
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class CryptonatorFeed(MarketFeed):
|
||||
@@ -195,7 +194,7 @@ class CryptonatorFeed(MarketFeed):
|
||||
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
|
||||
'success' not in json_response or json_response['success'] is not True:
|
||||
raise InvalidExchangeRateResponse(self.name, 'result not found')
|
||||
return defer.succeed(float(json_response['ticker']['price']))
|
||||
return float(json_response['ticker']['price'])
|
||||
|
||||
|
||||
class ExchangeRateManager:
|
||||
@@ -75,7 +75,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['channel_name'] = txo.channel.claim_name
|
||||
try:
|
||||
output['valid_signature'] = claim.validate_signature(
|
||||
txo.get_address(self.ledger), txo.channel.claim
|
||||
txo.get_address(self.ledger), txo.channel.claim, name=txo.claim_name
|
||||
)
|
||||
except BadSignatureError:
|
||||
output['valid_signature'] = False
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import logging.handlers
|
||||
import traceback
|
||||
from lbrynet import utils, __version__
|
||||
|
||||
|
||||
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
class HTTPSLogglyHandler(logging.Handler):
|
||||
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
|
||||
super().__init__()
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.cookies = cookies or {}
|
||||
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
|
||||
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
|
||||
)
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
async def _emit(self, record):
|
||||
payload = self.format(record)
|
||||
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
|
||||
def emit(self, record):
|
||||
asyncio.ensure_future(self._emit(record))
|
||||
|
||||
|
||||
def get_loggly_handler(loggly_token=LOGGLY_TOKEN):
|
||||
handler = HTTPSLogglyHandler(loggly_token)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
return handler
|
||||
@@ -3,7 +3,7 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def migrate_db(db_dir, start, end):
|
||||
def migrate_db(conf, start, end):
|
||||
current = start
|
||||
while current < end:
|
||||
if current == 1:
|
||||
@@ -22,12 +22,10 @@ def migrate_db(db_dir, start, end):
|
||||
from .migrate7to8 import do_migration
|
||||
elif current == 8:
|
||||
from .migrate8to9 import do_migration
|
||||
elif current == 9:
|
||||
from .migrate9to10 import do_migration
|
||||
else:
|
||||
raise Exception("DB migration of version {} to {} is not available".format(current,
|
||||
current+1))
|
||||
do_migration(db_dir)
|
||||
do_migration(conf)
|
||||
current += 1
|
||||
log.info("successfully migrated the database from revision %i to %i", current - 1, current)
|
||||
return None
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
UNSET_NOUT = -1
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blockchainname_db(db_dir)
|
||||
migrate_blockchainname_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blockchainname_db(db_dir)
|
||||
migrate_blockchainname_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
migrate_blobs_db(db_dir)
|
||||
migrate_blobs_db(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
def do_migration(conf):
|
||||
log.info("Doing the migration")
|
||||
add_lbry_file_metadata(db_dir)
|
||||
add_lbry_file_metadata(conf.data_dir)
|
||||
log.info("Migration succeeded")
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import sqlite3
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from lbrynet import conf
|
||||
from binascii import hexlify
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -103,13 +103,13 @@ def verify_sd_blob(sd_hash, blob_dir):
|
||||
return decoded, sd_length
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
new_db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
def do_migration(conf):
|
||||
new_db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(new_db_path)
|
||||
|
||||
metadata_db = sqlite3.connect(os.path.join(db_dir, "blockchainname.db"))
|
||||
lbryfile_db = sqlite3.connect(os.path.join(db_dir, 'lbryfile_info.db'))
|
||||
blobs_db = sqlite3.connect(os.path.join(db_dir, 'blobs.db'))
|
||||
metadata_db = sqlite3.connect(os.path.join(conf.data_dir, "blockchainname.db"))
|
||||
lbryfile_db = sqlite3.connect(os.path.join(conf.data_dir, 'lbryfile_info.db'))
|
||||
blobs_db = sqlite3.connect(os.path.join(conf.data_dir, 'blobs.db'))
|
||||
|
||||
name_metadata_cursor = metadata_db.cursor()
|
||||
lbryfile_cursor = lbryfile_db.cursor()
|
||||
@@ -185,10 +185,14 @@ def do_migration(db_dir):
|
||||
(stream_hash, blob_hash, position, iv)
|
||||
)
|
||||
|
||||
download_dir = conf.download_dir
|
||||
if not isinstance(download_dir, bytes):
|
||||
download_dir = download_dir.encode()
|
||||
|
||||
# insert the file
|
||||
transaction.execute(
|
||||
"insert or ignore into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, stream_name, conf.settings.download_dir.encode('hex'),
|
||||
(stream_hash, stream_name, hexlify(download_dir),
|
||||
data_rate, status)
|
||||
)
|
||||
|
||||
@@ -273,7 +277,7 @@ def do_migration(db_dir):
|
||||
|
||||
# recover damaged streams
|
||||
if damaged_stream_sds:
|
||||
blob_dir = os.path.join(db_dir, "blobfiles")
|
||||
blob_dir = os.path.join(conf.data_dir, "blobfiles")
|
||||
damaged_sds_on_disk = [] if not os.path.isdir(blob_dir) else list({p for p in os.listdir(blob_dir)
|
||||
if p in damaged_stream_sds})
|
||||
for damaged_sd in damaged_sds_on_disk:
|
||||
@@ -311,7 +315,7 @@ def do_migration(db_dir):
|
||||
log.warning("detected a failed previous migration to revision 6, repairing it")
|
||||
connection.close()
|
||||
os.remove(new_db_path)
|
||||
return do_migration(db_dir)
|
||||
return do_migration(conf)
|
||||
raise err
|
||||
|
||||
connection.close()
|
||||
|
||||
@@ -2,8 +2,8 @@ import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
cursor.executescript("alter table blob add last_announced_time integer;")
|
||||
|
||||
@@ -2,8 +2,8 @@ import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
|
||||
@@ -9,9 +9,9 @@ from lbrynet.blob.CryptBlob import CryptBlobInfo
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
blob_dir = os.path.join(db_dir, "blobfiles")
|
||||
def do_migration(conf):
|
||||
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
|
||||
blob_dir = os.path.join(conf.data_dir, "blobfiles")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
|
||||
def do_migration(db_dir):
|
||||
db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
connection = sqlite3.connect(db_path)
|
||||
cursor = connection.cursor()
|
||||
cursor.executescript("ALTER TABLE claim ADD resolved_at int DEFAULT 0 NOT NULL;")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
@@ -144,6 +144,7 @@ types_map = {
|
||||
'.pict': 'image/pict',
|
||||
'.rtf': 'application/rtf',
|
||||
'.xul': 'text/xul',
|
||||
'.m4v': 'video/m4v',
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
@@ -151,9 +152,7 @@ types_map = {
|
||||
}
|
||||
|
||||
|
||||
fallback_mime_type = 'application/octet-stream'
|
||||
|
||||
|
||||
def guess_mime_type(path):
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
return types_map.get(ext, fallback_mime_type)
|
||||
default = f'application/x-ext-{ext}' if ext else 'application/octet-stream'
|
||||
return types_map.get(ext, default)
|
||||
+157
-245
@@ -1,18 +1,16 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import traceback
|
||||
import typing
|
||||
from binascii import hexlify, unhexlify
|
||||
from twisted.internet import defer, task, threads
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbrynet import conf
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.blob.CryptBlob import CryptBlobInfo
|
||||
from lbrynet.dht.constants import dataExpireTimeout
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -45,63 +43,23 @@ def _open_file_for_writing(download_directory, suggested_file_name):
|
||||
return os.path.basename(file_path)
|
||||
|
||||
|
||||
def open_file_for_writing(download_directory, suggested_file_name):
|
||||
"""
|
||||
Used to touch the path of a file to be downloaded
|
||||
|
||||
:param download_directory: (str)
|
||||
:param suggested_file_name: (str)
|
||||
:return: (str) basename
|
||||
"""
|
||||
return threads.deferToThread(_open_file_for_writing, download_directory, suggested_file_name)
|
||||
async def open_file_for_writing(download_directory: str, suggested_file_name: str) -> str:
|
||||
""" Used to touch the path of a file to be downloaded. """
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
None, _open_file_for_writing, download_directory, suggested_file_name
|
||||
)
|
||||
|
||||
|
||||
def rerun_if_locked(f):
|
||||
max_attempts = 5
|
||||
|
||||
def rerun(err, rerun_count, *args, **kwargs):
|
||||
connection = args[0]
|
||||
reactor = connection.reactor
|
||||
log.debug("Failed to execute (%s): %s", err, args)
|
||||
if err.check(sqlite3.OperationalError) and "database is locked" in str(err.value):
|
||||
log.warning("database was locked. rerunning %s with args %s, kwargs %s",
|
||||
str(f), str(args), str(kwargs))
|
||||
if rerun_count < max_attempts:
|
||||
delay = 2**rerun_count
|
||||
return task.deferLater(reactor, delay, inner_wrapper, rerun_count + 1, *args, **kwargs)
|
||||
raise err
|
||||
|
||||
def check_needed_rerun(result, rerun_count):
|
||||
if rerun_count:
|
||||
log.info("successfully reran database query")
|
||||
return result
|
||||
|
||||
def inner_wrapper(rerun_count, *args, **kwargs):
|
||||
d = f(*args, **kwargs)
|
||||
d.addCallback(check_needed_rerun, rerun_count)
|
||||
d.addErrback(rerun, rerun_count, *args, **kwargs)
|
||||
return d
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
return inner_wrapper(0, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
async def looping_call(interval, fun):
|
||||
while True:
|
||||
try:
|
||||
await fun()
|
||||
except Exception as e:
|
||||
log.exception('Looping call experienced exception:', exc_info=e)
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
|
||||
class SqliteConnection(adbapi.ConnectionPool):
|
||||
def __init__(self, db_path):
|
||||
super().__init__('sqlite3', db_path, check_same_thread=False)
|
||||
|
||||
@rerun_if_locked
|
||||
def runInteraction(self, interaction, *args, **kw):
|
||||
return super().runInteraction(interaction, *args, **kw)
|
||||
|
||||
@classmethod
|
||||
def set_reactor(cls, reactor):
|
||||
cls.reactor = reactor
|
||||
|
||||
|
||||
class SQLiteStorage:
|
||||
class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
CREATE_TABLES_QUERY = """
|
||||
pragma foreign_keys=on;
|
||||
@@ -174,70 +132,47 @@ class SQLiteStorage:
|
||||
);
|
||||
"""
|
||||
|
||||
def __init__(self, db_dir, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.db_dir = db_dir
|
||||
self._db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
log.info("connecting to database: %s", self._db_path)
|
||||
self.db = SqliteConnection(self._db_path)
|
||||
self.db.set_reactor(reactor)
|
||||
self.clock = reactor
|
||||
|
||||
# used to refresh the claim attributes on a ManagedEncryptedFileDownloader when a
|
||||
# change to the associated content claim occurs. these are added by the file manager
|
||||
# when it loads each file
|
||||
self.content_claim_callbacks = {} # {<stream_hash>: <callable returning a deferred>}
|
||||
def __init__(self, conf: Config, path, loop=None):
|
||||
super().__init__(path)
|
||||
self.conf = conf
|
||||
self.content_claim_callbacks = {}
|
||||
self.check_should_announce_lc = None
|
||||
if 'reflector' not in conf.settings['components_to_skip']:
|
||||
self.check_should_announce_lc = task.LoopingCall(self.verify_will_announce_all_head_and_sd_blobs)
|
||||
self.loop = loop or asyncio.get_event_loop()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
def _create_tables(transaction):
|
||||
transaction.executescript(self.CREATE_TABLES_QUERY)
|
||||
yield self.db.runInteraction(_create_tables)
|
||||
if self.check_should_announce_lc and not self.check_should_announce_lc.running:
|
||||
self.check_should_announce_lc.start(600)
|
||||
defer.returnValue(None)
|
||||
async def open(self):
|
||||
await super().open()
|
||||
if 'reflector' not in self.conf.components_to_skip:
|
||||
self.check_should_announce_lc = looping_call(
|
||||
600, self.verify_will_announce_all_head_and_sd_blobs
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run_and_return_one_or_none(self, query, *args):
|
||||
result = yield self.db.runQuery(query, args)
|
||||
if result:
|
||||
defer.returnValue(result[0][0])
|
||||
else:
|
||||
defer.returnValue(None)
|
||||
async def close(self):
|
||||
if self.check_should_announce_lc is not None:
|
||||
self.check_should_announce_lc.close()
|
||||
await super().close()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run_and_return_list(self, query, *args):
|
||||
result = yield self.db.runQuery(query, args)
|
||||
if result:
|
||||
defer.returnValue([i[0] for i in result])
|
||||
else:
|
||||
defer.returnValue([])
|
||||
async def run_and_return_one_or_none(self, query, *args):
|
||||
for row in await self.db.execute_fetchall(query, args):
|
||||
if len(row) == 1:
|
||||
return row[0]
|
||||
return row
|
||||
|
||||
def run_and_return_id(self, query, *args):
|
||||
def do_save(t):
|
||||
t.execute(query, args)
|
||||
return t.lastrowid
|
||||
return self.db.runInteraction(do_save)
|
||||
async def run_and_return_list(self, query, *args):
|
||||
rows = list(await self.db.execute_fetchall(query, args))
|
||||
return [col[0] for col in rows] if rows else []
|
||||
|
||||
def stop(self):
|
||||
if self.check_should_announce_lc and self.check_should_announce_lc.running:
|
||||
self.check_should_announce_lc.stop()
|
||||
self.db.close()
|
||||
return defer.succeed(True)
|
||||
async def run_and_return_id(self, query, *args):
|
||||
return (await self.db.execute(query, args)).lastrowid
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
def add_completed_blob(self, blob_hash, length, next_announce_time, should_announce, status="finished"):
|
||||
log.debug("Adding a completed blob. blob_hash=%s, length=%i", blob_hash, length)
|
||||
values = (blob_hash, length, next_announce_time or 0, int(bool(should_announce)), status, 0, 0)
|
||||
return self.db.runOperation("insert or replace into blob values (?, ?, ?, ?, ?, ?, ?)", values)
|
||||
return self.db.execute("insert or replace into blob values (?, ?, ?, ?, ?, ?, ?)", values)
|
||||
|
||||
def set_should_announce(self, blob_hash, next_announce_time, should_announce):
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"update blob set next_announce_time=?, should_announce=? where blob_hash=?",
|
||||
(next_announce_time or 0, int(bool(should_announce)), blob_hash)
|
||||
)
|
||||
@@ -247,9 +182,8 @@ class SQLiteStorage:
|
||||
"select status from blob where blob_hash=?", blob_hash
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_known_blob(self, blob_hash, length):
|
||||
yield self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)", (blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
)
|
||||
|
||||
@@ -268,12 +202,11 @@ class SQLiteStorage:
|
||||
"select blob_hash from blob where should_announce=1 and status='finished'"
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_all_finished_blobs(self):
|
||||
blob_hashes = yield self.run_and_return_list(
|
||||
async def get_all_finished_blobs(self):
|
||||
blob_hashes = await self.run_and_return_list(
|
||||
"select blob_hash from blob where status='finished'"
|
||||
)
|
||||
defer.returnValue([unhexlify(blob_hash) for blob_hash in blob_hashes])
|
||||
return [unhexlify(blob_hash) for blob_hash in blob_hashes]
|
||||
|
||||
def count_finished_blobs(self):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -281,14 +214,14 @@ class SQLiteStorage:
|
||||
)
|
||||
|
||||
def update_last_announced_blob(self, blob_hash, last_announced):
|
||||
return self.db.runOperation(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 where blob_hash=?",
|
||||
(int(last_announced + (dataExpireTimeout / 2)), int(last_announced), blob_hash)
|
||||
)
|
||||
return self.db.execute(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 where blob_hash=?",
|
||||
(int(last_announced + (dataExpireTimeout / 2)), int(last_announced), blob_hash)
|
||||
)
|
||||
|
||||
def should_single_announce_blobs(self, blob_hashes, immediate=False):
|
||||
def set_single_announce(transaction):
|
||||
now = self.clock.seconds()
|
||||
now = self.loop.time()
|
||||
for blob_hash in blob_hashes:
|
||||
if immediate:
|
||||
transaction.execute(
|
||||
@@ -299,12 +232,12 @@ class SQLiteStorage:
|
||||
transaction.execute(
|
||||
"update blob set single_announce=1 where blob_hash=? and status='finished'", (blob_hash, )
|
||||
)
|
||||
return self.db.runInteraction(set_single_announce)
|
||||
return self.db.run(set_single_announce)
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
def get_and_update(transaction):
|
||||
timestamp = self.clock.seconds()
|
||||
if conf.settings['announce_head_blobs_only']:
|
||||
timestamp = self.loop.time()
|
||||
if self.conf.announce_head_blobs_only:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob "
|
||||
"where blob_hash is not null and "
|
||||
@@ -318,13 +251,13 @@ class SQLiteStorage:
|
||||
)
|
||||
blobs = [b[0] for b in r.fetchall()]
|
||||
return blobs
|
||||
return self.db.runInteraction(get_and_update)
|
||||
return self.db.run(get_and_update)
|
||||
|
||||
def delete_blobs_from_db(self, blob_hashes):
|
||||
def delete_blobs(transaction):
|
||||
for blob_hash in blob_hashes:
|
||||
transaction.execute("delete from blob where blob_hash=?;", (blob_hash,))
|
||||
return self.db.runInteraction(delete_blobs)
|
||||
return self.db.run(delete_blobs)
|
||||
|
||||
def get_all_blob_hashes(self):
|
||||
return self.run_and_return_list("select blob_hash from blob")
|
||||
@@ -337,17 +270,16 @@ class SQLiteStorage:
|
||||
transaction.execute("insert into stream_blob values (?, ?, ?, ?)",
|
||||
(stream_hash, blob_info.get('blob_hash', None),
|
||||
blob_info['blob_num'], blob_info['iv']))
|
||||
return self.db.runInteraction(_add_stream_blobs)
|
||||
return self.db.run(_add_stream_blobs)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_known_blobs(self, blob_infos):
|
||||
async def add_known_blobs(self, blob_infos):
|
||||
for blob_info in blob_infos:
|
||||
if blob_info.get('blob_hash') and blob_info['length']:
|
||||
yield self.add_known_blob(blob_info['blob_hash'], blob_info['length'])
|
||||
await self.add_known_blob(blob_info['blob_hash'], blob_info['length'])
|
||||
|
||||
def verify_will_announce_head_and_sd_blobs(self, stream_hash):
|
||||
# fix should_announce for imported head and sd blobs
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"update blob set should_announce=1 "
|
||||
"where should_announce=0 and "
|
||||
"blob.blob_hash in "
|
||||
@@ -359,7 +291,7 @@ class SQLiteStorage:
|
||||
)
|
||||
|
||||
def verify_will_announce_all_head_and_sd_blobs(self):
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"update blob set should_announce=1 "
|
||||
"where should_announce=0 and "
|
||||
"blob.blob_hash in "
|
||||
@@ -384,23 +316,24 @@ class SQLiteStorage:
|
||||
:param stream_blob_infos: (list) of blob info dictionaries
|
||||
:return: (defer.Deferred)
|
||||
"""
|
||||
|
||||
def _store_stream(transaction):
|
||||
transaction.execute("insert into stream values (?, ?, ?, ?, ?);",
|
||||
(stream_hash, sd_hash, stream_key, stream_name,
|
||||
suggested_file_name))
|
||||
|
||||
transaction.execute(
|
||||
"insert into stream values (?, ?, ?, ?, ?);", (
|
||||
stream_hash, sd_hash, stream_key, stream_name, suggested_file_name
|
||||
)
|
||||
)
|
||||
for blob_info in stream_blob_infos:
|
||||
transaction.execute("insert into stream_blob values (?, ?, ?, ?)",
|
||||
(stream_hash, blob_info.get('blob_hash', None),
|
||||
blob_info['blob_num'], blob_info['iv']))
|
||||
transaction.execute(
|
||||
"insert into stream_blob values (?, ?, ?, ?)", (
|
||||
stream_hash, blob_info.get('blob_hash', None),
|
||||
blob_info['blob_num'], blob_info['iv']
|
||||
)
|
||||
)
|
||||
return self.db.run(_store_stream)
|
||||
|
||||
return self.db.runInteraction(_store_stream)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_stream(self, stream_hash):
|
||||
sd_hash = yield self.get_sd_blob_hash_for_stream(stream_hash)
|
||||
stream_blobs = yield self.get_blobs_for_stream(stream_hash)
|
||||
async def delete_stream(self, stream_hash):
|
||||
sd_hash = await self.get_sd_blob_hash_for_stream(stream_hash)
|
||||
stream_blobs = await self.get_blobs_for_stream(stream_hash)
|
||||
blob_hashes = [b.blob_hash for b in stream_blobs if b.blob_hash is not None]
|
||||
|
||||
def _delete_stream(transaction):
|
||||
@@ -411,21 +344,25 @@ class SQLiteStorage:
|
||||
transaction.execute("delete from blob where blob_hash=?", (sd_hash, ))
|
||||
for blob_hash in blob_hashes:
|
||||
transaction.execute("delete from blob where blob_hash=?;", (blob_hash, ))
|
||||
yield self.db.runInteraction(_delete_stream)
|
||||
|
||||
await self.db.run(_delete_stream)
|
||||
|
||||
def get_all_streams(self):
|
||||
return self.run_and_return_list("select stream_hash from stream")
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
d = self.db.runQuery("select stream_name, stream_key, suggested_filename, sd_hash from stream "
|
||||
"where stream_hash=?", (stream_hash, ))
|
||||
d.addCallback(lambda r: None if not r else r[0])
|
||||
return d
|
||||
return self.run_and_return_one_or_none(
|
||||
"select stream_name, stream_key, suggested_filename, sd_hash from stream "
|
||||
"where stream_hash=?", stream_hash
|
||||
)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
d = self.db.runQuery("select stream_hash from stream where stream_hash=?", (stream_hash, ))
|
||||
d.addCallback(lambda r: bool(len(r)))
|
||||
return d
|
||||
async def check_if_stream_exists(self, stream_hash):
|
||||
row = await self.run_and_return_one_or_none(
|
||||
"select stream_hash from stream where stream_hash=?", stream_hash
|
||||
)
|
||||
if row is not None:
|
||||
return bool(len(row))
|
||||
return False
|
||||
|
||||
def get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -467,7 +404,7 @@ class SQLiteStorage:
|
||||
crypt_blob_infos.append(CryptBlobInfo(blob_hash, position, blob_length, iv))
|
||||
crypt_blob_infos = sorted(crypt_blob_infos, key=lambda info: info.blob_num)
|
||||
return crypt_blob_infos
|
||||
return self.db.runInteraction(_get_blobs_for_stream)
|
||||
return self.db.run(_get_blobs_for_stream)
|
||||
|
||||
def get_pending_blobs_for_stream(self, stream_hash):
|
||||
return self.run_and_return_list(
|
||||
@@ -494,14 +431,12 @@ class SQLiteStorage:
|
||||
|
||||
# # # # # # # # # file stuff # # # # # # # # #
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
|
||||
async def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
|
||||
# touch the closest available file to the file name
|
||||
file_name = yield open_file_for_writing(unhexlify(download_directory).decode(), unhexlify(file_name).decode())
|
||||
result = yield self.save_published_file(
|
||||
file_name = await open_file_for_writing(unhexlify(download_directory).decode(), unhexlify(file_name).decode())
|
||||
return await self.save_published_file(
|
||||
stream_hash, hexlify(file_name.encode()), download_directory, data_payment_rate
|
||||
)
|
||||
defer.returnValue(result)
|
||||
|
||||
def save_published_file(self, stream_hash, file_name, download_directory, data_payment_rate, status="stopped"):
|
||||
return self.run_and_return_id(
|
||||
@@ -510,7 +445,9 @@ class SQLiteStorage:
|
||||
)
|
||||
|
||||
def get_filename_for_rowid(self, rowid):
|
||||
return self.run_and_return_one_or_none("select file_name from file where rowid=?", rowid)
|
||||
return self.run_and_return_one_or_none(
|
||||
"select file_name from file where rowid=?", rowid
|
||||
)
|
||||
|
||||
def get_all_lbry_files(self):
|
||||
def _lbry_file_dict(rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
@@ -536,13 +473,11 @@ class SQLiteStorage:
|
||||
).fetchall()
|
||||
]
|
||||
|
||||
d = self.db.runInteraction(_get_all_files)
|
||||
return d
|
||||
return self.db.run(_get_all_files)
|
||||
|
||||
def change_file_status(self, rowid, new_status):
|
||||
d = self.db.runQuery("update file set status=? where rowid=?", (new_status, rowid))
|
||||
d.addCallback(lambda _: new_status)
|
||||
return d
|
||||
async def change_file_status(self, rowid, new_status):
|
||||
await self.db.execute("update file set status=? where rowid=?", (new_status, rowid))
|
||||
return new_status
|
||||
|
||||
def get_lbry_file_status(self, rowid):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -566,14 +501,16 @@ class SQLiteStorage:
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
return self.db.runInteraction(_save_support)
|
||||
return self.db.run(_save_support)
|
||||
|
||||
def get_supports(self, *claim_ids):
|
||||
def _format_support(outpoint, amount):
|
||||
def _format_support(outpoint, supported_id, amount, address):
|
||||
return {
|
||||
"txid": outpoint.split(":")[0],
|
||||
"nout": int(outpoint.split(":")[1]),
|
||||
"claim_id": supported_id,
|
||||
"amount": dewies_to_lbc(amount),
|
||||
"address": address,
|
||||
}
|
||||
|
||||
def _get_supports(transaction):
|
||||
@@ -581,20 +518,21 @@ class SQLiteStorage:
|
||||
_format_support(*support_info)
|
||||
for support_info in _batched_select(
|
||||
transaction,
|
||||
"select support_outpoint, amount from support where claim_id in {}",
|
||||
"select * from support where claim_id in {}",
|
||||
tuple(claim_ids)
|
||||
)
|
||||
]
|
||||
|
||||
return self.db.runInteraction(_get_supports)
|
||||
return self.db.run(_get_supports)
|
||||
|
||||
# # # # # # # # # claim functions # # # # # # # # #
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def save_claims(self, claim_infos):
|
||||
async def save_claims(self, claim_infos):
|
||||
support_callbacks = []
|
||||
update_file_callbacks = []
|
||||
|
||||
def _save_claims(transaction):
|
||||
content_claims_to_update = []
|
||||
support_callbacks = []
|
||||
for claim_info in claim_infos:
|
||||
outpoint = "%s:%i" % (claim_info['txid'], claim_info['nout'])
|
||||
claim_id = claim_info['claim_id']
|
||||
@@ -603,7 +541,6 @@ class SQLiteStorage:
|
||||
height = claim_info['height']
|
||||
address = claim_info['address']
|
||||
sequence = claim_info['claim_sequence']
|
||||
resolved_at = claim_info['height'] + claim_info['depth']
|
||||
try:
|
||||
certificate_id = claim_info['value'].get('publisherSignature', {}).get('certificateId')
|
||||
except AttributeError:
|
||||
@@ -617,13 +554,12 @@ class SQLiteStorage:
|
||||
source_hash = None
|
||||
serialized = claim_info.get('hex') or hexlify(smart_decode(claim_info['value']).serialized)
|
||||
transaction.execute(
|
||||
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence,
|
||||
resolved_at)
|
||||
"insert or replace into claim values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(outpoint, claim_id, name, amount, height, serialized, certificate_id, address, sequence)
|
||||
)
|
||||
if 'supports' in claim_info: # if this response doesn't have support info don't overwrite the existing
|
||||
# support info
|
||||
support_callbacks.append(self.save_supports(claim_id, claim_info['supports']))
|
||||
support_callbacks.append((claim_id, claim_info['supports']))
|
||||
if not source_hash:
|
||||
continue
|
||||
stream_hash = transaction.execute(
|
||||
@@ -645,18 +581,18 @@ class SQLiteStorage:
|
||||
content_claims_to_update.append((stream_hash, outpoint))
|
||||
elif known_outpoint != outpoint:
|
||||
content_claims_to_update.append((stream_hash, outpoint))
|
||||
update_file_callbacks = []
|
||||
for stream_hash, outpoint in content_claims_to_update:
|
||||
self._save_content_claim(transaction, outpoint, stream_hash)
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
update_file_callbacks.append(self.content_claim_callbacks[stream_hash]())
|
||||
return update_file_callbacks, support_callbacks
|
||||
|
||||
content_dl, support_dl = yield self.db.runInteraction(_save_claims)
|
||||
if content_dl:
|
||||
yield defer.DeferredList(content_dl)
|
||||
if support_dl:
|
||||
yield defer.DeferredList(support_dl)
|
||||
await self.db.run(_save_claims)
|
||||
if update_file_callbacks:
|
||||
await asyncio.wait(update_file_callbacks)
|
||||
if support_callbacks:
|
||||
await asyncio.wait([
|
||||
self.save_supports(*args) for args in support_callbacks
|
||||
])
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = []
|
||||
@@ -719,16 +655,13 @@ class SQLiteStorage:
|
||||
# update the claim associated to the file
|
||||
transaction.execute("insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def save_content_claim(self, stream_hash, claim_outpoint):
|
||||
yield self.db.runInteraction(self._save_content_claim, claim_outpoint, stream_hash)
|
||||
async def save_content_claim(self, stream_hash, claim_outpoint):
|
||||
await self.db.run(self._save_content_claim, claim_outpoint, stream_hash)
|
||||
# update corresponding ManagedEncryptedFileDownloader object
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
file_callback = self.content_claim_callbacks[stream_hash]
|
||||
yield file_callback()
|
||||
await self.content_claim_callbacks[stream_hash]()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_content_claim(self, stream_hash, include_supports=True):
|
||||
async def get_content_claim(self, stream_hash, include_supports=True):
|
||||
def _get_claim_from_stream_hash(transaction):
|
||||
claim_info = transaction.execute(
|
||||
"select c.*, "
|
||||
@@ -746,15 +679,13 @@ class SQLiteStorage:
|
||||
result['channel_name'] = channel_name
|
||||
return result
|
||||
|
||||
result = yield self.db.runInteraction(_get_claim_from_stream_hash)
|
||||
result = await self.db.run(_get_claim_from_stream_hash)
|
||||
if result and include_supports:
|
||||
supports = yield self.get_supports(result['claim_id'])
|
||||
result['supports'] = supports
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], supports)
|
||||
defer.returnValue(result)
|
||||
result['supports'] = await self.get_supports(result['claim_id'])
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], result['supports'])
|
||||
return result
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claims_from_stream_hashes(self, stream_hashes, include_supports=True):
|
||||
async def get_claims_from_stream_hashes(self, stream_hashes, include_supports=True):
|
||||
def _batch_get_claim(transaction):
|
||||
results = {}
|
||||
claim_infos = _batched_select(
|
||||
@@ -782,10 +713,10 @@ class SQLiteStorage:
|
||||
results[stream_hash]['channel_name'] = channel_name
|
||||
return results
|
||||
|
||||
claims = yield self.db.runInteraction(_batch_get_claim)
|
||||
claims = await self.db.run(_batch_get_claim)
|
||||
if include_supports:
|
||||
all_supports = {}
|
||||
for support in (yield self.get_supports(*[claim['claim_id'] for claim in claims.values()])):
|
||||
for support in await self.get_supports(*[claim['claim_id'] for claim in claims.values()]):
|
||||
all_supports.setdefault(support['claim_id'], []).append(support)
|
||||
for stream_hash in claims.keys():
|
||||
claim = claims[stream_hash]
|
||||
@@ -793,46 +724,28 @@ class SQLiteStorage:
|
||||
claim['supports'] = supports
|
||||
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
|
||||
claims[stream_hash] = claim
|
||||
defer.returnValue(claims)
|
||||
return claims
|
||||
|
||||
def get_resolved_at_for_outpoint(self, outpoint):
|
||||
return self.run_and_return_one_or_none(
|
||||
"select resolved_at from claim "
|
||||
"where claim_outpoint = ?",
|
||||
outpoint
|
||||
)
|
||||
|
||||
def get_outpoint_for_uri(self, uri):
|
||||
uri = parse_lbry_uri(uri)
|
||||
claim_id_param = "%" if not uri.claim_id else uri.claim_id
|
||||
return self.run_and_return_one_or_none(
|
||||
"select claim_outpoint from claim "
|
||||
"where claim_name=? and claim_id like ?"
|
||||
"order by height desc, amount desc "
|
||||
"limit 1",
|
||||
uri.name, claim_id_param
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claim(self, claim_outpoint, include_supports=True):
|
||||
async def get_claim(self, claim_outpoint, include_supports=True):
|
||||
def _get_claim(transaction):
|
||||
claim_info = transaction.execute("select c.*, "
|
||||
"case when c.channel_claim_id is not null then "
|
||||
"(select claim_name from claim where claim_id==c.channel_claim_id) "
|
||||
"else null end as channel_name from claim c where claim_outpoint = ?",
|
||||
(claim_outpoint,)).fetchone()
|
||||
claim_info = transaction.execute(
|
||||
"select c.*, "
|
||||
"case when c.channel_claim_id is not null then "
|
||||
"(select claim_name from claim where claim_id==c.channel_claim_id) "
|
||||
"else null end as channel_name from claim c where claim_outpoint = ?",
|
||||
(claim_outpoint,)
|
||||
).fetchone()
|
||||
channel_name = claim_info[-1]
|
||||
result = _format_claim_response(*claim_info[:-1])
|
||||
if channel_name:
|
||||
result['channel_name'] = channel_name
|
||||
return result
|
||||
|
||||
result = yield self.db.runInteraction(_get_claim)
|
||||
result = await self.db.run(_get_claim)
|
||||
if include_supports:
|
||||
supports = yield self.get_supports(result['claim_id'])
|
||||
result['supports'] = supports
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], supports)
|
||||
defer.returnValue(result)
|
||||
result['supports'] = await self.get_supports(result['claim_id'])
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], result['supports'])
|
||||
return result
|
||||
|
||||
def get_unknown_certificate_ids(self):
|
||||
def _get_unknown_certificate_claim_ids(transaction):
|
||||
@@ -844,11 +757,10 @@ class SQLiteStorage:
|
||||
"(select c2.claim_id from claim as c2)"
|
||||
).fetchall()
|
||||
]
|
||||
return self.db.runInteraction(_get_unknown_certificate_claim_ids)
|
||||
return self.db.run(_get_unknown_certificate_claim_ids)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_pending_claim_outpoints(self):
|
||||
claim_outpoints = yield self.run_and_return_list("select claim_outpoint from claim where height=-1")
|
||||
async def get_pending_claim_outpoints(self):
|
||||
claim_outpoints = await self.run_and_return_list("select claim_outpoint from claim where height=-1")
|
||||
results = {} # {txid: [nout, ...]}
|
||||
for outpoint_str in claim_outpoints:
|
||||
txid, nout = outpoint_str.split(":")
|
||||
@@ -857,7 +769,7 @@ class SQLiteStorage:
|
||||
results[txid] = outputs
|
||||
if results:
|
||||
log.debug("missing transaction heights for %i claims", len(results))
|
||||
defer.returnValue(results)
|
||||
return results
|
||||
|
||||
def save_claim_tx_heights(self, claim_tx_heights):
|
||||
def _save_claim_heights(transaction):
|
||||
@@ -866,17 +778,17 @@ class SQLiteStorage:
|
||||
"update claim set height=? where claim_outpoint=? and height=-1",
|
||||
(height, outpoint)
|
||||
)
|
||||
return self.db.runInteraction(_save_claim_heights)
|
||||
return self.db.run(_save_claim_heights)
|
||||
|
||||
# # # # # # # # # reflector functions # # # # # # # # #
|
||||
|
||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||
if success:
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.clock.seconds())
|
||||
(sd_hash, reflector_address, self.loop.time())
|
||||
)
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
(sd_hash, reflector_address)
|
||||
)
|
||||
@@ -886,24 +798,24 @@ class SQLiteStorage:
|
||||
"select s.sd_hash from stream s "
|
||||
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
|
||||
"where r.timestamp is null or r.timestamp < ?",
|
||||
self.clock.seconds() - conf.settings['auto_re_reflect_interval']
|
||||
self.loop.time() - self.conf.auto_re_reflect_interval
|
||||
)
|
||||
|
||||
|
||||
# Helper functions
|
||||
def _format_claim_response(outpoint, claim_id, name, amount, height, serialized, _channel_id, address, claim_sequence,
|
||||
_resolved_at):
|
||||
def _format_claim_response(outpoint, claim_id, name, amount, height, serialized, channel_id, address, claim_sequence):
|
||||
r = {
|
||||
"name": name,
|
||||
"claim_id": claim_id,
|
||||
"address": address,
|
||||
"claim_sequence": claim_sequence,
|
||||
"hex": serialized,
|
||||
"value": ClaimDict.deserialize(unhexlify(serialized)).claim_dict,
|
||||
"height": height,
|
||||
"amount": dewies_to_lbc(amount),
|
||||
"nout": int(outpoint.split(":")[1]),
|
||||
"txid": outpoint.split(":")[0],
|
||||
"channel_claim_id": channel_id,
|
||||
"channel_name": None
|
||||
}
|
||||
return r
|
||||
|
||||
|
||||
File renamed without changes.
@@ -165,15 +165,10 @@ class BlobReflectorClient(Protocol):
|
||||
blob_hash = self.blob_hashes_to_send[0]
|
||||
log.debug('No current blob, sending the next one: %s', blob_hash)
|
||||
self.blob_hashes_to_send = self.blob_hashes_to_send[1:]
|
||||
d = self.blob_manager.get_blob(blob_hash)
|
||||
d.addCallback(self.open_blob_for_reading)
|
||||
blob = self.blob_manager.get_blob(blob_hash)
|
||||
self.open_blob_for_reading(blob)
|
||||
# send the server the next blob hash + length
|
||||
d.addCallbacks(
|
||||
lambda _: self.send_blob_info(),
|
||||
errback=log.fail(self.disconnect),
|
||||
errbackArgs=("Error reflecting blob %s", blob_hash)
|
||||
)
|
||||
return d
|
||||
return self.send_blob_info()
|
||||
else:
|
||||
# close connection
|
||||
log.debug('No more blob hashes, closing connection')
|
||||
|
||||
@@ -6,6 +6,7 @@ from twisted.protocols.basic import FileSender
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.internet import defer, error
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.reflector.common import IncompleteResponse, ReflectorRequestError
|
||||
from lbrynet.extras.reflector.common import REFLECTOR_V1, REFLECTOR_V2
|
||||
|
||||
@@ -61,7 +62,9 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
else:
|
||||
reflected = False
|
||||
|
||||
d = self.blob_manager.storage.update_reflected_stream(self.sd_hash, self.transport.getPeer().host, reflected)
|
||||
d = f2d(self.blob_manager.storage.update_reflected_stream(
|
||||
self.sd_hash, self.transport.getPeer().host, reflected
|
||||
))
|
||||
d.addCallback(lambda _: result)
|
||||
return d
|
||||
|
||||
@@ -113,10 +116,7 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
for crypt_blob in blobs:
|
||||
if crypt_blob.blob_hash and crypt_blob.length:
|
||||
yield self.blob_manager.get_blob(crypt_blob.blob_hash, crypt_blob.length)
|
||||
|
||||
dl = defer.DeferredList(list(get_blobs(blobs_in_stream)), consumeErrors=True)
|
||||
dl.addCallback(lambda blobs: [blob for r, blob in blobs if r and blob.get_is_verified()])
|
||||
return dl
|
||||
return [blob for blob in get_blobs(blobs_in_stream) if blob.get_is_verified()]
|
||||
|
||||
def set_blobs_to_send(self, blobs_to_send):
|
||||
for blob in blobs_to_send:
|
||||
@@ -132,7 +132,7 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
len(filtered))
|
||||
return filtered
|
||||
|
||||
d = self.factory.blob_manager.storage.get_blobs_for_stream(self.stream_hash)
|
||||
d = f2d(self.factory.blob_manager.storage.get_blobs_for_stream(self.stream_hash))
|
||||
d.addCallback(self.get_validated_blobs)
|
||||
if not self.descriptor_needed:
|
||||
d.addCallback(lambda filtered:
|
||||
@@ -303,7 +303,7 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
# open the next blob to send
|
||||
blob_hash = self.blob_hashes_to_send[0]
|
||||
self.blob_hashes_to_send = self.blob_hashes_to_send[1:]
|
||||
d = self.blob_manager.get_blob(blob_hash)
|
||||
d = defer.succeed(self.blob_manager.get_blob(blob_hash))
|
||||
d.addCallback(self.open_blob_for_reading)
|
||||
d.addCallbacks(lambda _: self.send_blob_info(),
|
||||
lambda err: self.skip_missing_blob(err, blob.blob_hash))
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
import random
|
||||
|
||||
from twisted.internet import reactor, defer
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory
|
||||
from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory
|
||||
|
||||
@@ -45,43 +42,22 @@ def _reflect_blobs(blob_manager, blob_hashes, reflector_server):
|
||||
ip = yield resolve(reflector_address)
|
||||
yield reactor.connectTCP(ip, reflector_port, factory)
|
||||
result = yield factory.finished_deferred
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
|
||||
|
||||
def reflect_file(lbry_file, reflector_server=None):
|
||||
if reflector_server:
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = reflector_server, 5566
|
||||
def reflect_file(lbry_file, reflector_server):
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = random.choice(conf.settings['reflector_servers'])
|
||||
reflector_server = reflector_server, 5566
|
||||
return _reflect_file(lbry_file, reflector_server)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def reflect_stream(blob_manager, stream_hash, reflector_server=None):
|
||||
if reflector_server:
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = reflector_server, 5566
|
||||
def reflect_blob_hashes(blob_hashes, blob_manager, reflector_server):
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = random.choice(conf.settings['reflector_servers'])
|
||||
sd_hash = yield blob_manager.storage.get_sd_blob_hash_for_stream(stream_hash)
|
||||
result = yield _reflect_stream(blob_manager, stream_hash, sd_hash, reflector_server)
|
||||
defer.returnValue(result)
|
||||
|
||||
|
||||
def reflect_blob_hashes(blob_hashes, blob_manager, reflector_server=None):
|
||||
if reflector_server:
|
||||
if len(reflector_server.split(":")) == 2:
|
||||
host, port = tuple(reflector_server.split(":"))
|
||||
reflector_server = host, int(port)
|
||||
else:
|
||||
reflector_server = reflector_server, 5566
|
||||
else:
|
||||
reflector_server = random.choice(conf.settings['reflector_servers'])
|
||||
reflector_server = reflector_server, 5566
|
||||
return _reflect_blobs(blob_manager, blob_hashes, reflector_server)
|
||||
@@ -236,3 +236,6 @@ class Account(BaseAccount):
|
||||
)
|
||||
|
||||
return tx
|
||||
|
||||
async def release_all_outputs(self):
|
||||
await self.ledger.db.release_all_outputs(self)
|
||||
@@ -117,3 +117,11 @@ class WalletDatabase(BaseDatabase):
|
||||
channel.private_key = private_key
|
||||
certificates.append(channel)
|
||||
return certificates
|
||||
|
||||
async def release_all_outputs(self, account):
|
||||
await self.db.execute(
|
||||
"UPDATE txo SET is_reserved = 0 WHERE"
|
||||
" is_reserved = 1 AND txo.address IN ("
|
||||
" SELECT address from pubkey_address WHERE account = ?"
|
||||
" )", [account.public_key.address]
|
||||
)
|
||||
@@ -30,4 +30,4 @@ def lbc_to_dewies(lbc: str) -> int:
|
||||
|
||||
|
||||
def dewies_to_lbc(dewies) -> str:
|
||||
return dewies if isinstance(dewies, str) else satoshis_to_coins(dewies)
|
||||
return satoshis_to_coins(dewies)
|
||||
@@ -1,6 +1,5 @@
|
||||
import os
|
||||
import json
|
||||
import asyncio
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
|
||||
@@ -10,12 +9,12 @@ from typing import Optional
|
||||
from twisted.internet import defer
|
||||
|
||||
from lbrynet.schema.schema import SECP256k1
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from torba.client.basemanager import BaseWalletManager
|
||||
from torba.rpc.jsonrpc import CodeMessageError
|
||||
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.wallet.ledger import MainNetLedger
|
||||
from lbrynet.extras.wallet.account import BaseAccount, generate_certificate
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
@@ -26,10 +25,6 @@ from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def d2f(deferred):
|
||||
return deferred.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
class ReservedPoints:
|
||||
def __init__(self, identifier, amount):
|
||||
self.identifier = identifier
|
||||
@@ -171,14 +166,12 @@ class LbryWalletManager(BaseWalletManager):
|
||||
'lbrycrd_main': 'lbc_mainnet',
|
||||
'lbrycrd_testnet': 'lbc_testnet',
|
||||
'lbrycrd_regtest': 'lbc_regtest'
|
||||
}[settings['blockchain_name']]
|
||||
}[settings.blockchain_name]
|
||||
|
||||
ledger_config = {
|
||||
'auto_connect': True,
|
||||
'default_servers': settings['lbryum_servers'],
|
||||
'default_servers': settings.lbryum_servers,
|
||||
'data_path': settings.wallet_dir,
|
||||
'use_keyring': settings['use_keyring'],
|
||||
#'db': db
|
||||
}
|
||||
|
||||
wallets_directory = os.path.join(settings.wallet_dir, 'wallets')
|
||||
@@ -273,55 +266,16 @@ class LbryWalletManager(BaseWalletManager):
|
||||
def get_info_exchanger(self):
|
||||
return LBRYcrdAddressRequester(self)
|
||||
|
||||
async def fetch_from_cache(self, uri):
|
||||
resolver = self.ledger.resolver
|
||||
is_channel = parse_lbry_uri(uri).is_channel
|
||||
|
||||
claim_outpoint = await d2f(self.old_db.get_outpoint_for_uri(uri))
|
||||
last_resolved_at = await d2f(self.old_db.get_resolved_at_for_outpoint(claim_outpoint))
|
||||
|
||||
if not last_resolved_at or resolver.height - last_resolved_at > 1:
|
||||
return False
|
||||
|
||||
log.info("trying to fetch %s from cache", uri)
|
||||
claim = await d2f(self.old_db.get_claim(claim_outpoint))
|
||||
claim['depth'] = resolver.height - claim['height']
|
||||
result = {"claim": claim}
|
||||
|
||||
if claim.get("channel_name", False):
|
||||
certificate_uri = "{}#{}".format(claim.get("channel_name", ""),
|
||||
claim['value'].get('publisherSignature', {}).get('certificateId'))
|
||||
certificate_outpoint = await d2f(self.old_db.get_outpoint_for_uri(certificate_uri))
|
||||
certificate = await d2f(self.old_db.get_claim(certificate_outpoint))
|
||||
certificate['depth'] = resolver.height - certificate['height']
|
||||
result["certificate"] = await resolver.parse_and_validate_claim_result(certificate)
|
||||
|
||||
result["claim"] = await resolver.parse_and_validate_claim_result(result["claim"],
|
||||
certificate=result.get(
|
||||
"certificate", None))
|
||||
if is_channel:
|
||||
result = {"certificate": result["claim"]}
|
||||
return result
|
||||
|
||||
async def resolve(self, *uris, **kwargs):
|
||||
page = kwargs.get('page', 0)
|
||||
page_size = kwargs.get('page_size', 10)
|
||||
check_cache = kwargs.get('check_cache', False) # TODO: put caching back (was force_refresh parameter)
|
||||
|
||||
cached_results = dict()
|
||||
needed = list()
|
||||
if check_cache:
|
||||
for uri in uris:
|
||||
cached_results[uri] = await self.fetch_from_cache(uri)
|
||||
needed = [key for key in cached_results if not cached_results[key]]
|
||||
|
||||
ledger: MainNetLedger = self.default_account.ledger
|
||||
results = await ledger.resolve(page, page_size, *needed)
|
||||
results = await ledger.resolve(page, page_size, *uris)
|
||||
if 'error' not in results:
|
||||
await self.old_db.save_claims_for_resolve([
|
||||
value for value in results.values() if 'error' not in value
|
||||
]).asFuture(asyncio.get_event_loop())
|
||||
results = {**cached_results, **results}
|
||||
])
|
||||
return results
|
||||
|
||||
async def get_claims_for_name(self, name: str):
|
||||
@@ -457,7 +411,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
claim_address = await account.receiving.get_or_create_usable_address()
|
||||
if certificate:
|
||||
claim = claim.sign(
|
||||
certificate.private_key, claim_address, certificate.claim_id, curve=SECP256k1
|
||||
certificate.private_key, claim_address, certificate.claim_id, curve=SECP256k1, name=name
|
||||
)
|
||||
existing_claims = await account.get_claims(
|
||||
claim_name_type__any={'is_claim': 1, 'is_update': 1}, # exclude is_supports
|
||||
@@ -476,7 +430,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.old_db.save_claims([self._old_get_temp_claim_info(
|
||||
tx, tx.outputs[0], claim_address, claim_dict, name, dewies_to_lbc(amount)
|
||||
)]).asFuture(asyncio.get_event_loop())
|
||||
)])
|
||||
# TODO: release reserved tx outputs in case anything fails by this point
|
||||
return tx
|
||||
|
||||
@@ -490,7 +444,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
'address': holding_address,
|
||||
'claim_id': claim_id,
|
||||
'amount': dewies_to_lbc(amount)
|
||||
}]).asFuture(asyncio.get_event_loop())
|
||||
}])
|
||||
return tx
|
||||
|
||||
async def tip_claim(self, amount, claim_id, account):
|
||||
@@ -505,7 +459,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
'address': claim_to_tip['address'],
|
||||
'claim_id': claim_id,
|
||||
'amount': dewies_to_lbc(amount)
|
||||
}]).asFuture(asyncio.get_event_loop())
|
||||
}])
|
||||
return tx
|
||||
|
||||
async def abandon_claim(self, claim_id, txid, nout, account):
|
||||
@@ -528,7 +482,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
|
||||
await self.old_db.save_claims([self._old_get_temp_claim_info(
|
||||
tx, tx.outputs[0], address, cert, channel_name, dewies_to_lbc(amount)
|
||||
)]).asFuture(asyncio.get_event_loop())
|
||||
)])
|
||||
return tx
|
||||
|
||||
def _old_get_temp_claim_info(self, tx, txo, address, claim_dict, name, bid):
|
||||
@@ -655,8 +609,7 @@ class LBRYcrdAddressQueryHandler:
|
||||
@defer.inlineCallbacks
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
future = self.wallet.get_unused_address_for_peer(self.peer)
|
||||
address = yield defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
address = yield f2d(self.wallet.get_unused_address_for_peer(self.peer))
|
||||
self.address = address
|
||||
fields = {'lbrycrd_address': address}
|
||||
return fields
|
||||
|
||||
@@ -155,8 +155,8 @@ class Resolver:
|
||||
certificate = None
|
||||
certificate_id = smart_decode(claim_result['value']).certificate_id
|
||||
if certificate_id:
|
||||
certificate = await self.network.get_claims_by_ids(certificate_id.decode())
|
||||
certificate = certificate.pop(certificate_id.decode()) if certificate else None
|
||||
certificate = await self.network.get_claims_by_ids(certificate_id)
|
||||
certificate = certificate.pop(certificate_id) if certificate else None
|
||||
return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)
|
||||
|
||||
async def parse_and_validate_claim_result(self, claim_result, certificate=None, raw=False):
|
||||
@@ -180,14 +180,13 @@ class Resolver:
|
||||
if decoded.has_signature:
|
||||
if certificate is None:
|
||||
log.info("fetching certificate to check claim signature")
|
||||
certificate_id = decoded.certificate_id.decode()
|
||||
certificate = (await self.network.get_claims_by_ids(certificate_id)).get(certificate_id, {})
|
||||
certificate = await self.network.get_claims_by_ids(decoded.certificate_id)
|
||||
if not certificate:
|
||||
log.warning('Certificate %s not found', decoded.certificate_id)
|
||||
claim_result['has_signature'] = True
|
||||
claim_result['signature_is_valid'] = False
|
||||
validated, channel_name = validate_claim_signature_and_get_channel_name(
|
||||
decoded, certificate, claim_result['address'])
|
||||
decoded, certificate, claim_result['address'], claim_result['name'])
|
||||
claim_result['channel_name'] = channel_name
|
||||
if validated:
|
||||
claim_result['signature_is_valid'] = True
|
||||
@@ -198,7 +197,7 @@ class Resolver:
|
||||
if 'amount' in claim_result:
|
||||
claim_result = format_amount_value(claim_result)
|
||||
|
||||
claim_result['permanent_url'] = _get_permanent_url(claim_result)
|
||||
claim_result['permanent_url'] = _get_permanent_url(claim_result, decoded.certificate_id)
|
||||
|
||||
return claim_result
|
||||
|
||||
@@ -299,11 +298,6 @@ def format_amount_value(obj):
|
||||
if not isinstance(obj[k], float):
|
||||
obj[k] = dewies_to_lbc(obj[k])
|
||||
elif k == 'supports' and isinstance(v, list):
|
||||
# supports are already in the desired format
|
||||
# the following 2 lines are bad as supports should be passed in a consistent manner
|
||||
# but it is unavoidable at the moment because of so much other code linked to this :-(
|
||||
if v and isinstance(v[0], dict):
|
||||
continue
|
||||
obj[k] = [{'txid': txid, 'nout': nout, 'amount': dewies_to_lbc(amount)}
|
||||
for (txid, nout, amount) in v]
|
||||
elif isinstance(v, (list, dict)):
|
||||
@@ -313,18 +307,11 @@ def format_amount_value(obj):
|
||||
return obj
|
||||
|
||||
|
||||
def _get_permanent_url(claim_result):
|
||||
if claim_result.get('has_signature') and claim_result.get('channel_name'):
|
||||
return "{}#{}/{}".format(
|
||||
claim_result['channel_name'],
|
||||
claim_result['value']['publisherSignature']['certificateId'],
|
||||
claim_result['name']
|
||||
)
|
||||
def _get_permanent_url(claim_result, certificate_id):
|
||||
if certificate_id:
|
||||
return f"{claim_result['channel_name']}#{certificate_id}/{claim_result['name']}"
|
||||
else:
|
||||
return "{}#{}".format(
|
||||
claim_result['name'],
|
||||
claim_result['claim_id']
|
||||
)
|
||||
return f"{claim_result['name']}#{claim_result['claim_id']}"
|
||||
|
||||
|
||||
def _verify_proof(name, claim_trie_root, result, height, depth, transaction_class, hash160_to_address):
|
||||
@@ -392,7 +379,7 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
|
||||
|
||||
|
||||
def validate_claim_signature_and_get_channel_name(claim, certificate_claim,
|
||||
claim_address, decoded_certificate=None):
|
||||
claim_address, name, decoded_certificate=None):
|
||||
if not certificate_claim:
|
||||
return False, None
|
||||
if 'value' not in certificate_claim:
|
||||
@@ -401,18 +388,18 @@ def validate_claim_signature_and_get_channel_name(claim, certificate_claim,
|
||||
certificate = decoded_certificate or smart_decode(certificate_claim['value'])
|
||||
if not isinstance(certificate, ClaimDict):
|
||||
raise TypeError("Certificate is not a ClaimDict: %s" % str(type(certificate)))
|
||||
if _validate_signed_claim(claim, claim_address, certificate):
|
||||
if _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
return True, certificate_claim['name']
|
||||
return False, None
|
||||
|
||||
|
||||
def _validate_signed_claim(claim, claim_address, certificate):
|
||||
def _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
if not claim.has_signature:
|
||||
raise Exception("Claim is not signed")
|
||||
if not is_address(claim_address):
|
||||
raise Exception("Not given a valid claim address")
|
||||
try:
|
||||
if claim.validate_signature(claim_address, certificate.protobuf):
|
||||
if claim.validate_signature(claim_address, certificate.protobuf, name):
|
||||
return True
|
||||
except BadSignatureError:
|
||||
# print_msg("Signature for %s is invalid" % claim_id)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import hashlib
|
||||
import struct
|
||||
from binascii import unhexlify
|
||||
|
||||
import msgpack
|
||||
from torba.server.hash import hash_to_hex_str
|
||||
|
||||
from torba.server.block_processor import BlockProcessor
|
||||
from lbrynet.schema.proto.claim_pb2 import Claim
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
|
||||
@@ -131,9 +131,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
super().backup_blocks(raw_blocks=raw_blocks)
|
||||
self.db.batched_flush_claims()
|
||||
|
||||
def shutdown(self):
|
||||
self.db.shutdown()
|
||||
|
||||
async def flush(self, flush_utxos):
|
||||
self.db.batched_flush_claims()
|
||||
return await super().flush(flush_utxos)
|
||||
@@ -153,14 +150,14 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
def _checksig(self, name, value, address):
|
||||
try:
|
||||
parse_lbry_uri(name.decode()) # skip invalid names
|
||||
cert_id = Claim.FromString(value).publisherSignature.certificateId[::-1] or None
|
||||
claim_dict = smart_decode(value)
|
||||
cert_id = unhexlify(claim_dict.certificate_id)[::-1]
|
||||
if not self.should_validate_signatures:
|
||||
return cert_id
|
||||
if cert_id:
|
||||
cert_claim = self.db.get_claim_info(cert_id)
|
||||
if cert_claim:
|
||||
certificate = smart_decode(cert_claim.value)
|
||||
claim_dict = smart_decode(value)
|
||||
claim_dict.validate_signature(address, certificate)
|
||||
return cert_id
|
||||
except Exception as e:
|
||||
|
||||
@@ -137,3 +137,8 @@ class LBCRegTest(LBC):
|
||||
XPRV_VERBYTES = bytes.fromhex('04358394')
|
||||
P2PKH_VERBYTE = bytes.fromhex("6f")
|
||||
P2SH_VERBYTES = bytes.fromhex("c4")
|
||||
|
||||
|
||||
class LBCTestNet(LBCRegTest):
|
||||
NET = "testnet"
|
||||
GENESIS_HASH = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
|
||||
@@ -37,7 +37,7 @@ class LBCDaemon(Daemon):
|
||||
@handles_errors
|
||||
async def getclaimsfortx(self, txid):
|
||||
'''Given a txid, returns the claims it make.'''
|
||||
return await self._send_single('getclaimsfortx', (txid,))
|
||||
return await self._send_single('getclaimsfortx', (txid,)) or []
|
||||
|
||||
@handles_errors
|
||||
async def getnameproof(self, name, block_hash=None):
|
||||
|
||||
@@ -21,7 +21,7 @@ class LBRYDB(DB):
|
||||
self.pending_abandons = {}
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def shutdown(self):
|
||||
def close(self):
|
||||
self.batched_flush_claims()
|
||||
self.claims_db.close()
|
||||
self.names_db.close()
|
||||
@@ -29,9 +29,7 @@ class LBRYDB(DB):
|
||||
self.outpoint_to_claim_id_db.close()
|
||||
self.claim_undo_db.close()
|
||||
self.utxo_db.close()
|
||||
# electrumx ones
|
||||
self.utxo_db.close()
|
||||
self.history.close_db()
|
||||
super().close()
|
||||
|
||||
async def _open_dbs(self, for_sync, compacting):
|
||||
await super()._open_dbs(for_sync=for_sync, compacting=compacting)
|
||||
|
||||
@@ -307,7 +307,7 @@ class LBRYElectrumX(ElectrumX):
|
||||
return {'error': err.message}
|
||||
result = {}
|
||||
|
||||
if parsed_uri.is_channel:
|
||||
if parsed_uri.contains_channel:
|
||||
certificate = None
|
||||
|
||||
# TODO: this is also done on the else, refactor
|
||||
|
||||
@@ -1,43 +1,11 @@
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import sys
|
||||
import traceback
|
||||
import treq
|
||||
import twisted.python.log
|
||||
from twisted.internet import defer
|
||||
from lbrynet import __version__ as lbrynet_version, build_type
|
||||
from lbrynet import utils, conf
|
||||
|
||||
|
||||
class HTTPSHandler(logging.Handler):
|
||||
def __init__(self, url, fqdn=False, localname=None, facility=None, cookies=None):
|
||||
super().__init__()
|
||||
self.url = url
|
||||
self.fqdn = fqdn
|
||||
self.localname = localname
|
||||
self.facility = facility
|
||||
self.cookies = cookies or {}
|
||||
|
||||
def get_full_message(self, record):
|
||||
if record.exc_info:
|
||||
return '\n'.join(traceback.format_exception(*record.exc_info))
|
||||
else:
|
||||
return record.getMessage()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _emit(self, record):
|
||||
payload = self.format(record)
|
||||
response = yield treq.post(self.url, data=payload.encode(), cookies=self.cookies)
|
||||
self.cookies.update(response.cookies())
|
||||
|
||||
def emit(self, record):
|
||||
return self._emit(record)
|
||||
|
||||
|
||||
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
|
||||
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
|
||||
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
|
||||
|
||||
|
||||
def remove_handlers(log, handler_name):
|
||||
@@ -108,63 +76,6 @@ def configure_file_handler(file_name, **kwargs):
|
||||
return handler
|
||||
|
||||
|
||||
def get_loggly_url(token=None, version=None):
|
||||
token = token or utils.deobfuscate(conf.settings['LOGGLY_TOKEN'])
|
||||
version = version or lbrynet_version
|
||||
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
|
||||
|
||||
|
||||
def configure_loggly_handler():
|
||||
if build_type.BUILD == 'dev':
|
||||
return
|
||||
if not conf.settings['share_usage_data']:
|
||||
return
|
||||
level = logging.ERROR
|
||||
handler = get_loggly_handler(level=level, installation_id=conf.settings.installation_id,
|
||||
session_id=conf.settings.get_session_id())
|
||||
log = logging.getLogger("lbrynet")
|
||||
if handler.name:
|
||||
remove_handlers(log, handler.name)
|
||||
handler.setLevel(level)
|
||||
log.addHandler(handler)
|
||||
# need to reduce the logger's level down to the
|
||||
# handler's level or else the handler won't
|
||||
# get those messages
|
||||
if log.level > level:
|
||||
log.setLevel(level)
|
||||
|
||||
|
||||
def get_loggly_handler(level, installation_id, session_id):
|
||||
formatter = JsonFormatter(level=level, installation_id=installation_id, session_id=session_id)
|
||||
handler = HTTPSHandler(get_loggly_url())
|
||||
handler.setFormatter(formatter)
|
||||
handler.name = 'loggly'
|
||||
return handler
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Format log records using json serialization"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.attributes = kwargs
|
||||
|
||||
def format(self, record):
|
||||
data = {
|
||||
'loggerName': record.name,
|
||||
'asciTime': self.formatTime(record),
|
||||
'fileName': record.filename,
|
||||
'functionName': record.funcName,
|
||||
'levelNo': record.levelno,
|
||||
'lineNo': record.lineno,
|
||||
'levelName': record.levelname,
|
||||
'message': record.getMessage(),
|
||||
}
|
||||
data.update(self.attributes)
|
||||
if record.exc_info:
|
||||
data['exc_info'] = self.formatException(record.exc_info)
|
||||
return json.dumps(data)
|
||||
|
||||
|
||||
def failure(failure, log, msg, *args):
|
||||
"""Log a failure message from a deferred.
|
||||
|
||||
|
||||
+31
-36
@@ -1,8 +1,10 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from binascii import unhexlify
|
||||
from sqlite3 import IntegrityError
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.internet import defer
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.creator import BlobFileCreator
|
||||
|
||||
@@ -26,15 +28,13 @@ class DiskBlobManager:
|
||||
self.blobs = {}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
async def setup(self):
|
||||
if self._node_datastore is not None:
|
||||
raw_blob_hashes = yield self.storage.get_all_finished_blobs()
|
||||
raw_blob_hashes = await self.storage.get_all_finished_blobs()
|
||||
self._node_datastore.completed_blobs.update(raw_blob_hashes)
|
||||
defer.returnValue(True)
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(True)
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
def get_blob(self, blob_hash, length=None):
|
||||
"""Return a blob identified by blob_hash, which may be a new blob or a
|
||||
@@ -43,7 +43,7 @@ class DiskBlobManager:
|
||||
if length is not None and not isinstance(length, int):
|
||||
raise Exception("invalid length type: {} ({})".format(length, str(type(length))))
|
||||
if blob_hash in self.blobs:
|
||||
return defer.succeed(self.blobs[blob_hash])
|
||||
return self.blobs[blob_hash]
|
||||
return self._make_new_blob(blob_hash, length)
|
||||
|
||||
def get_blob_creator(self):
|
||||
@@ -53,13 +53,13 @@ class DiskBlobManager:
|
||||
log.debug('Making a new blob for %s', blob_hash)
|
||||
blob = BlobFile(self.blob_dir, blob_hash, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
return defer.succeed(blob)
|
||||
return blob
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def blob_completed(self, blob, should_announce=False, next_announce_time=None):
|
||||
yield self.storage.add_completed_blob(
|
||||
yield f2d(self.storage.add_completed_blob(
|
||||
blob.blob_hash, blob.length, next_announce_time, should_announce
|
||||
)
|
||||
))
|
||||
if self._node_datastore is not None:
|
||||
self._node_datastore.completed_blobs.add(unhexlify(blob.blob_hash))
|
||||
|
||||
@@ -67,14 +67,15 @@ class DiskBlobManager:
|
||||
return self._completed_blobs(blobhashes_to_check)
|
||||
|
||||
def count_should_announce_blobs(self):
|
||||
return self.storage.count_should_announce_blobs()
|
||||
return f2d(self.storage.count_should_announce_blobs())
|
||||
|
||||
def set_should_announce(self, blob_hash, should_announce):
|
||||
now = self.storage.clock.seconds()
|
||||
return self.storage.set_should_announce(blob_hash, now, should_announce)
|
||||
return f2d(self.storage.set_should_announce(
|
||||
blob_hash, asyncio.get_event_loop().time(), should_announce
|
||||
))
|
||||
|
||||
def get_should_announce(self, blob_hash):
|
||||
return self.storage.should_announce(blob_hash)
|
||||
return f2d(self.storage.should_announce(blob_hash))
|
||||
|
||||
def creator_finished(self, blob_creator, should_announce):
|
||||
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
@@ -89,7 +90,7 @@ class DiskBlobManager:
|
||||
return self.blob_completed(new_blob, should_announce)
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d = f2d(self._get_all_verified_blob_hashes())
|
||||
d.addCallback(self.completed_blobs)
|
||||
return d
|
||||
|
||||
@@ -105,35 +106,29 @@ class DiskBlobManager:
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
blob = yield self.get_blob(blob_hash)
|
||||
yield blob.delete()
|
||||
blob = self.get_blob(blob_hash)
|
||||
blob.delete()
|
||||
bh_to_delete_from_db.append(blob_hash)
|
||||
del self.blobs[blob_hash]
|
||||
except Exception as e:
|
||||
log.warning("Failed to delete blob file. Reason: %s", e)
|
||||
try:
|
||||
yield self.storage.delete_blobs_from_db(bh_to_delete_from_db)
|
||||
yield f2d(self.storage.delete_blobs_from_db(bh_to_delete_from_db))
|
||||
except IntegrityError as err:
|
||||
if str(err) != "FOREIGN KEY constraint failed":
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _completed_blobs(self, blobhashes_to_check):
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = yield defer.DeferredList([self.get_blob(b) for b in blobhashes_to_check])
|
||||
blob_hashes = [b.blob_hash for success, b in blobs if success and b.verified]
|
||||
defer.returnValue(blob_hashes)
|
||||
blobs = [self.get_blob(b) for b in blobhashes_to_check]
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.verified]
|
||||
return blob_hashes
|
||||
|
||||
def _get_all_verified_blob_hashes(self):
|
||||
d = self.storage.get_all_blob_hashes()
|
||||
|
||||
def get_verified_blobs(blobs):
|
||||
verified_blobs = []
|
||||
for blob_hash in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
|
||||
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
|
||||
return d
|
||||
async def _get_all_verified_blob_hashes(self):
|
||||
blobs = await self.storage.get_all_blob_hashes()
|
||||
verified_blobs = []
|
||||
for blob_hash in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
@@ -5,7 +5,7 @@ from twisted.internet import defer, task
|
||||
from twisted.internet.error import ConnectingCancelledError
|
||||
from twisted.web._newclient import ResponseNeverReceived
|
||||
|
||||
from lbrynet.utils import DeferredDict
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.Error import DownloadCanceledError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -104,9 +104,7 @@ class HTTPBlobDownloader:
|
||||
log.debug("trying to download stream from mirror (sd %s)", self.sd_hashes[0][:8])
|
||||
else:
|
||||
log.debug("trying to download %i blobs from mirror", len(self.blob_hashes))
|
||||
blobs = yield DeferredDict(
|
||||
{blob_hash: self.blob_manager.get_blob(blob_hash) for blob_hash in self.blob_hashes}
|
||||
)
|
||||
blobs = {blob_hash: self.blob_manager.get_blob(blob_hash) for blob_hash in self.blob_hashes}
|
||||
self.deferreds = [self.download_blob(blobs[blob_hash]) for blob_hash in self.blob_hashes]
|
||||
yield defer.DeferredList(self.deferreds)
|
||||
if self.retry and self.missing_blob_hashes:
|
||||
@@ -159,7 +157,7 @@ class HTTPBlobDownloader:
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _write_blob(self, writer, blob):
|
||||
response = yield self.client.get(url_for(choice(self.servers), blob.blob_hash))
|
||||
response = yield self.client.get(url_for('{}:{}'.format(*choice(self.servers)), blob.blob_hash))
|
||||
if response.code != 200:
|
||||
log.debug('Missing a blob: %s', blob.blob_hash)
|
||||
if blob.blob_hash in self.blob_hashes:
|
||||
@@ -175,7 +173,7 @@ class HTTPBlobDownloader:
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_stream(self, stream_hash, sd_hash):
|
||||
stream_crypt_blobs = yield self.blob_manager.storage.get_blobs_for_stream(stream_hash)
|
||||
stream_crypt_blobs = yield f2d(self.blob_manager.storage.get_blobs_for_stream(stream_hash))
|
||||
self.blob_hashes.extend([
|
||||
b.blob_hash for b in stream_crypt_blobs
|
||||
if b.blob_hash and b.blob_hash not in self.blob_hashes
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
from decimal import Decimal
|
||||
from lbrynet import conf
|
||||
from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy
|
||||
|
||||
|
||||
class BasePaymentRateManager:
|
||||
def __init__(self, rate=None, info_rate=None):
|
||||
self.min_blob_data_payment_rate = rate if rate is not None else conf.settings['data_rate']
|
||||
self.min_blob_info_payment_rate = (
|
||||
info_rate if info_rate is not None else conf.settings['min_info_rate'])
|
||||
def __init__(self, rate, info_rate):
|
||||
self.min_blob_data_payment_rate = rate
|
||||
self.min_blob_info_payment_rate = info_rate
|
||||
|
||||
|
||||
class PaymentRateManager:
|
||||
@@ -37,7 +35,7 @@ class PaymentRateManager:
|
||||
|
||||
|
||||
class NegotiatedPaymentRateManager:
|
||||
def __init__(self, base, availability_tracker, generous=None):
|
||||
def __init__(self, base, availability_tracker, generous):
|
||||
"""
|
||||
@param base: a BasePaymentRateManager
|
||||
@param availability_tracker: a BlobAvailabilityTracker
|
||||
@@ -48,10 +46,10 @@ class NegotiatedPaymentRateManager:
|
||||
self.min_blob_data_payment_rate = base.min_blob_data_payment_rate
|
||||
self.points_paid = 0.0
|
||||
self.blob_tracker = availability_tracker
|
||||
self.generous = generous if generous is not None else conf.settings['is_generous_host']
|
||||
self.strategy = get_default_strategy(self.blob_tracker,
|
||||
base_price=self.base.min_blob_data_payment_rate,
|
||||
is_generous=generous)
|
||||
self.generous = generous
|
||||
self.strategy = get_default_strategy(
|
||||
self.blob_tracker, self.base.min_blob_data_payment_rate, generous
|
||||
)
|
||||
|
||||
def get_rate_blob_data(self, peer, blobs):
|
||||
response = self.strategy.make_offer(peer, blobs)
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
from decimal import Decimal
|
||||
from lbrynet import conf
|
||||
|
||||
|
||||
def get_default_price_model(blob_tracker, base_price, **kwargs):
|
||||
@@ -26,9 +25,8 @@ class MeanAvailabilityWeightedPrice:
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, tracker, base_price=None, alpha=1.0):
|
||||
def __init__(self, tracker, base_price, alpha=1.0):
|
||||
self.blob_tracker = tracker
|
||||
base_price = base_price if base_price is not None else conf.settings['data_rate']
|
||||
self.base_price = Decimal(base_price)
|
||||
self.alpha = Decimal(alpha)
|
||||
|
||||
|
||||
@@ -2,8 +2,9 @@ import logging
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from twisted.internet import defer, threads, reactor
|
||||
from twisted.internet import defer, reactor
|
||||
|
||||
from lbrynet.extras.compat import d2f
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.p2p.BlobManager import DiskBlobManager
|
||||
from lbrynet.p2p.RateLimiter import DummyRateLimiter
|
||||
@@ -13,6 +14,7 @@ from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder
|
||||
from lbrynet.conf import Config
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -59,7 +61,8 @@ class SingleBlobDownloadManager:
|
||||
|
||||
|
||||
class SinglePeerDownloader:
|
||||
def __init__(self):
|
||||
def __init__(self, conf: Config):
|
||||
self.conf = conf
|
||||
self._payment_rate_manager = OnlyFreePaymentsManager()
|
||||
self._rate_limiter = DummyRateLimiter()
|
||||
self._wallet = None
|
||||
@@ -80,7 +83,7 @@ class SinglePeerDownloader:
|
||||
peer_finder = SinglePeerFinder(peer)
|
||||
requester = BlobRequester(blob_manager, peer_finder, self._payment_rate_manager,
|
||||
self._wallet, download_manager)
|
||||
downloader = StandaloneBlobDownloader(blob_hash, blob_manager, peer_finder,
|
||||
downloader = StandaloneBlobDownloader(self.conf, blob_hash, blob_manager, peer_finder,
|
||||
self._rate_limiter, self._payment_rate_manager,
|
||||
self._wallet, timeout=timeout)
|
||||
info_exchanger = self._wallet.get_info_exchanger()
|
||||
@@ -91,19 +94,16 @@ class SinglePeerDownloader:
|
||||
result = yield blob.callback
|
||||
if not result:
|
||||
log.debug("Failed to downloaded %s from %s", blob_hash[:16], peer.host)
|
||||
yield connection_manager.stop()
|
||||
yield connection_manager.stop()
|
||||
defer.returnValue(result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
|
||||
tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
|
||||
tmp_storage = SQLiteStorage(tmp_dir)
|
||||
yield tmp_storage.setup()
|
||||
async def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
|
||||
tmp_storage = SQLiteStorage(Config(), ':memory:')
|
||||
await tmp_storage.open()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
tmp_blob_manager = DiskBlobManager(tmp_dir, tmp_storage)
|
||||
try:
|
||||
result = yield self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager)
|
||||
return await d2f(self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager))
|
||||
finally:
|
||||
yield tmp_blob_manager.stop()
|
||||
yield tmp_storage.stop()
|
||||
yield threads.deferToThread(shutil.rmtree, tmp_dir)
|
||||
defer.returnValue(result)
|
||||
await tmp_storage.close()
|
||||
shutil.rmtree(tmp_dir)
|
||||
+8
-12
@@ -1,11 +1,10 @@
|
||||
from decimal import Decimal
|
||||
from lbrynet import conf
|
||||
from lbrynet.p2p.Offer import Offer
|
||||
from lbrynet.p2p.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
|
||||
|
||||
|
||||
def get_default_strategy(blob_tracker, **kwargs):
|
||||
return BasicAvailabilityWeightedStrategy(blob_tracker, **kwargs)
|
||||
def get_default_strategy(blob_tracker, base_price, is_generous, **kwargs):
|
||||
return BasicAvailabilityWeightedStrategy(blob_tracker, base_price, is_generous, **kwargs)
|
||||
|
||||
|
||||
class Strategy:
|
||||
@@ -13,10 +12,9 @@ class Strategy:
|
||||
Base for negotiation strategies
|
||||
"""
|
||||
|
||||
def __init__(self, price_model, max_rate, min_rate, is_generous=None):
|
||||
def __init__(self, price_model, max_rate, min_rate, is_generous):
|
||||
self.price_model = price_model
|
||||
self.is_generous = (
|
||||
is_generous if is_generous is not None else conf.settings['is_generous_host'])
|
||||
self.is_generous = is_generous
|
||||
self.accepted_offers = {}
|
||||
self.pending_sent_offers = {}
|
||||
self.offers_sent = {}
|
||||
@@ -98,13 +96,11 @@ class BasicAvailabilityWeightedStrategy(Strategy):
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, blob_tracker, acceleration=1.25,
|
||||
deceleration=0.9, max_rate=None,
|
||||
min_rate=0.0,
|
||||
is_generous=None,
|
||||
base_price=0.0001, alpha=1.0):
|
||||
def __init__(self, blob_tracker, base_price, is_generous,
|
||||
acceleration=1.25, deceleration=0.9, max_rate=None,
|
||||
min_rate=0.0, alpha=1.0):
|
||||
price_model = MeanAvailabilityWeightedPrice(
|
||||
blob_tracker, base_price=base_price, alpha=alpha)
|
||||
blob_tracker, base_price, alpha=alpha)
|
||||
super().__init__(price_model, max_rate, min_rate, is_generous)
|
||||
self._acceleration = Decimal(acceleration) # rate of how quickly to ramp offer
|
||||
self._deceleration = Decimal(deceleration)
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections import defaultdict
|
||||
from binascii import unhexlify
|
||||
from twisted.internet import threads, defer
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.p2p.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
|
||||
@@ -260,7 +261,7 @@ def save_sd_info(blob_manager, sd_hash, sd_info):
|
||||
if calculated_sd_hash != sd_hash:
|
||||
raise InvalidStreamDescriptorError("%s does not match calculated %s" %
|
||||
(sd_hash, calculated_sd_hash))
|
||||
stream_hash = yield blob_manager.storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
stream_hash = yield f2d(blob_manager.storage.get_stream_hash_for_sd_hash(sd_hash))
|
||||
if not stream_hash:
|
||||
log.debug("Saving info for %s", unhexlify(sd_info['stream_name']))
|
||||
stream_name = sd_info['stream_name']
|
||||
@@ -268,10 +269,10 @@ def save_sd_info(blob_manager, sd_hash, sd_info):
|
||||
stream_hash = sd_info['stream_hash']
|
||||
stream_blobs = sd_info['blobs']
|
||||
suggested_file_name = sd_info['suggested_file_name']
|
||||
yield blob_manager.storage.add_known_blobs(stream_blobs)
|
||||
yield blob_manager.storage.store_stream(
|
||||
yield f2d(blob_manager.storage.add_known_blobs(stream_blobs))
|
||||
yield f2d(blob_manager.storage.store_stream(
|
||||
stream_hash, sd_hash, stream_name, key, suggested_file_name, stream_blobs
|
||||
)
|
||||
))
|
||||
defer.returnValue(stream_hash)
|
||||
|
||||
|
||||
@@ -299,8 +300,7 @@ def format_sd_info(stream_type, stream_name, key, suggested_file_name, stream_ha
|
||||
}
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_sd_info(storage, stream_hash, include_blobs):
|
||||
async def get_sd_info(storage, stream_hash, include_blobs):
|
||||
"""
|
||||
Get an sd info dictionary from storage
|
||||
|
||||
@@ -329,16 +329,13 @@ def get_sd_info(storage, stream_hash, include_blobs):
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
stream_info = yield storage.get_stream_info(stream_hash)
|
||||
stream_info = await storage.get_stream_info(stream_hash)
|
||||
blobs = []
|
||||
if include_blobs:
|
||||
blobs = yield storage.get_blobs_for_stream(stream_hash)
|
||||
defer.returnValue(
|
||||
format_sd_info(
|
||||
EncryptedFileStreamType, stream_info[0], stream_info[1],
|
||||
stream_info[2], stream_hash, format_blobs(blobs)
|
||||
)
|
||||
blobs = await storage.get_blobs_for_stream(stream_hash)
|
||||
return format_sd_info(
|
||||
EncryptedFileStreamType, stream_info[0], stream_info[1],
|
||||
stream_info[2], stream_hash, format_blobs(blobs)
|
||||
)
|
||||
|
||||
|
||||
@@ -432,8 +429,8 @@ class EncryptedFileStreamDescriptorValidator:
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_sd_blob(blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet, timeout=None,
|
||||
download_mirrors=None):
|
||||
def download_sd_blob(conf, blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet,
|
||||
timeout=None, download_mirrors=None):
|
||||
"""
|
||||
Downloads a single blob from the network
|
||||
|
||||
@@ -446,7 +443,8 @@ def download_sd_blob(blob_hash, blob_manager, peer_finder, rate_limiter, payment
|
||||
@return: An object of type HashBlob
|
||||
"""
|
||||
|
||||
downloader = StandaloneBlobDownloader(blob_hash,
|
||||
downloader = StandaloneBlobDownloader(conf,
|
||||
blob_hash,
|
||||
blob_manager,
|
||||
peer_finder,
|
||||
rate_limiter,
|
||||
@@ -465,6 +463,6 @@ def download_sd_blob(blob_hash, blob_manager, peer_finder, rate_limiter, payment
|
||||
yield blob_manager.delete_blobs([blob_hash])
|
||||
raise err
|
||||
raw_sd = yield sd_reader._get_raw_data()
|
||||
yield blob_manager.storage.add_known_blob(blob_hash, len(raw_sd))
|
||||
yield f2d(blob_manager.storage.add_known_blob(blob_hash, len(raw_sd)))
|
||||
yield save_sd_info(blob_manager, sd_blob.blob_hash, sd_info)
|
||||
defer.returnValue(sd_blob)
|
||||
@@ -5,7 +5,8 @@ from twisted.internet import error, defer
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.protocols.policies import TimeoutMixin
|
||||
from twisted.python import failure
|
||||
from lbrynet import conf, utils
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import MAX_RESPONSE_INFO_SIZE
|
||||
from lbrynet.p2p.Error import ConnectionClosedBeforeResponseError, NoResponseError
|
||||
from lbrynet.p2p.Error import DownloadCanceledError, MisbehavingPeerError
|
||||
from lbrynet.p2p.Error import RequestCanceledError
|
||||
@@ -52,7 +53,7 @@ class ClientProtocol(Protocol, TimeoutMixin):
|
||||
self._blob_download_request.write(data)
|
||||
else:
|
||||
self._response_buff += data
|
||||
if len(self._response_buff) > conf.settings['MAX_RESPONSE_INFO_SIZE']:
|
||||
if len(self._response_buff) > MAX_RESPONSE_INFO_SIZE:
|
||||
log.warning("Response is too large from %s. Size %s",
|
||||
self.peer, len(self._response_buff))
|
||||
self.transport.loseConnection()
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import random
|
||||
import logging
|
||||
from twisted.internet import defer, reactor
|
||||
from lbrynet import utils, conf
|
||||
from lbrynet import utils
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.p2p.client.ClientProtocol import ClientProtocolFactory
|
||||
from lbrynet.p2p.Error import InsufficientFundsError
|
||||
|
||||
@@ -20,11 +21,11 @@ class ConnectionManager:
|
||||
MANAGE_CALL_INTERVAL_SEC = 5
|
||||
TCP_CONNECT_TIMEOUT = 15
|
||||
|
||||
def __init__(self, downloader, rate_limiter,
|
||||
primary_request_creators, secondary_request_creators):
|
||||
def __init__(self, downloader, rate_limiter, primary_request_creators, secondary_request_creators):
|
||||
|
||||
self.seek_head_blob_first = conf.settings['seek_head_blob_first']
|
||||
self.max_connections_per_stream = conf.settings['max_connections_per_stream']
|
||||
self.conf: Config = downloader.conf
|
||||
self.seek_head_blob_first = self.conf.seek_head_blob_first
|
||||
self.max_connections_per_stream = self.conf.max_connections_per_stream
|
||||
|
||||
self.downloader = downloader
|
||||
self.rate_limiter = rate_limiter
|
||||
|
||||
@@ -30,7 +30,7 @@ class DownloadManager:
|
||||
def resume_downloading(self):
|
||||
yield self.connection_manager.start()
|
||||
yield self.progress_manager.start()
|
||||
defer.returnValue(True)
|
||||
return True
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop_downloading(self):
|
||||
@@ -39,32 +39,14 @@ class DownloadManager:
|
||||
defer.returnValue(True)
|
||||
|
||||
def add_blobs_to_download(self, blob_infos):
|
||||
|
||||
log.debug("Adding %s blobs to blobs", len(blob_infos))
|
||||
|
||||
def add_blob_to_list(blob, blob_num):
|
||||
self.blobs[blob_num] = blob
|
||||
log.debug(
|
||||
"Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_num)
|
||||
|
||||
def error_during_add(err):
|
||||
log.warning(
|
||||
"An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
ds = []
|
||||
for blob_info in blob_infos:
|
||||
if not blob_info.blob_num in self.blobs:
|
||||
self.blob_infos[blob_info.blob_num] = blob_info
|
||||
log.debug(
|
||||
"Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
d.addCallback(add_blob_to_list, blob_info.blob_num)
|
||||
d.addErrback(error_during_add)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
return dl
|
||||
log.debug("Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
|
||||
blob = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
self.blobs[blob_info.blob_num] = blob
|
||||
log.debug("Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_info.blob_num)
|
||||
|
||||
def stream_position(self):
|
||||
return self.progress_manager.stream_position()
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
from lbrynet.conf import Config
|
||||
from lbrynet.p2p.BlobInfo import BlobInfo
|
||||
from lbrynet.p2p.client.BlobRequester import BlobRequester
|
||||
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
|
||||
@@ -82,9 +83,10 @@ class DummyBlobHandler:
|
||||
|
||||
|
||||
class StandaloneBlobDownloader:
|
||||
def __init__(self, blob_hash, blob_manager, peer_finder,
|
||||
def __init__(self, conf: Config, blob_hash, blob_manager, peer_finder,
|
||||
rate_limiter, payment_rate_manager, wallet,
|
||||
timeout=None):
|
||||
self.conf = conf
|
||||
self.blob_hash = blob_hash
|
||||
self.blob_manager = blob_manager
|
||||
self.peer_finder = peer_finder
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -39,19 +38,12 @@ class BlobAvailabilityHandler:
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
log.info("Received the client's list of requested blobs")
|
||||
d = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
|
||||
def set_field(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
|
||||
d.addCallback(set_field)
|
||||
return d
|
||||
return defer.succeed({})
|
||||
available_blobs = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
return {}
|
||||
|
||||
######### internal #########
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
|
||||
return d
|
||||
return self.blob_manager.completed_blobs(requested_blobs)
|
||||
@@ -61,21 +61,21 @@ class BlobRequestHandler:
|
||||
request_handler.register_blob_sender(self)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
response = defer.succeed({})
|
||||
response = {}
|
||||
log.debug("Handle query: %s", str(queries))
|
||||
|
||||
if self.AVAILABILITY_QUERY in queries:
|
||||
self._blobs_requested = queries[self.AVAILABILITY_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_availability(r, self._blobs_requested))
|
||||
self._reply_to_availability(response, self._blobs_requested)
|
||||
if self.PAYMENT_RATE_QUERY in queries:
|
||||
offered_rate = queries[self.PAYMENT_RATE_QUERY]
|
||||
offer = Offer(offered_rate)
|
||||
if offer.rate is None:
|
||||
log.warning("Empty rate offer")
|
||||
response.addCallback(lambda r: self._handle_payment_rate_query(offer, r))
|
||||
self._handle_payment_rate_query(offer, response)
|
||||
if self.BLOB_QUERY in queries:
|
||||
incoming = queries[self.BLOB_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_send_request(r, incoming))
|
||||
self._reply_to_send_request(response, incoming)
|
||||
return response
|
||||
|
||||
######### IBlobSender #########
|
||||
@@ -95,15 +95,10 @@ class BlobRequestHandler:
|
||||
######### internal #########
|
||||
|
||||
def _reply_to_availability(self, request, blobs):
|
||||
d = self._get_available_blobs(blobs)
|
||||
|
||||
def set_available(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
request.update({'available_blobs': available_blobs})
|
||||
return request
|
||||
|
||||
d.addCallback(set_available)
|
||||
return d
|
||||
available_blobs = self._get_available_blobs(blobs)
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
request.update({'available_blobs': available_blobs})
|
||||
return request
|
||||
|
||||
def _handle_payment_rate_query(self, offer, request):
|
||||
blobs = self._blobs_requested
|
||||
@@ -172,8 +167,7 @@ class BlobRequestHandler:
|
||||
return d
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
return d
|
||||
return self.blob_manager.completed_blobs(requested_blobs)
|
||||
|
||||
def send_file(self, consumer):
|
||||
|
||||
|
||||
@@ -167,7 +167,7 @@ class ServerRequestHandler:
|
||||
ds = []
|
||||
for query_handler, query_identifiers in self.query_handlers.items():
|
||||
queries = {q_i: msg[q_i] for q_i in query_identifiers if q_i in msg}
|
||||
d = query_handler.handle_queries(queries)
|
||||
d = defer.maybeDeferred(query_handler.handle_queries(queries))
|
||||
d.addErrback(log_errors)
|
||||
ds.append(d)
|
||||
|
||||
|
||||
+30
-25
@@ -7,19 +7,21 @@ from collections import OrderedDict
|
||||
|
||||
from lbrynet.schema.schema.claim import Claim
|
||||
from lbrynet.schema.proto import claim_pb2
|
||||
from lbrynet.schema.signature import Signature
|
||||
from lbrynet.schema.validator import get_validator
|
||||
from lbrynet.schema.signer import get_signer
|
||||
from lbrynet.schema.schema import NIST256p, CURVE_NAMES, CLAIM_TYPE_NAMES
|
||||
from lbrynet.schema.schema import NIST256p, CURVE_NAMES, CLAIM_TYPE_NAMES, SECP256k1
|
||||
from lbrynet.schema.encoding import decode_fields, decode_b64_fields, encode_fields
|
||||
from lbrynet.schema.error import DecodeError
|
||||
from lbrynet.schema.fee import Fee
|
||||
|
||||
|
||||
class ClaimDict(OrderedDict):
|
||||
def __init__(self, claim_dict=None):
|
||||
def __init__(self, claim_dict=None, detached_signature: Signature=None):
|
||||
if isinstance(claim_dict, claim_pb2.Claim):
|
||||
raise Exception("To initialize %s with a Claim protobuf use %s.load_protobuf" %
|
||||
(self.__class__.__name__, self.__class__.__name__))
|
||||
self.detached_signature = detached_signature
|
||||
OrderedDict.__init__(self, claim_dict or [])
|
||||
|
||||
@property
|
||||
@@ -37,7 +39,8 @@ class ClaimDict(OrderedDict):
|
||||
@property
|
||||
def serialized(self):
|
||||
"""Serialized Claim protobuf"""
|
||||
|
||||
if self.detached_signature:
|
||||
return self.detached_signature.serialized
|
||||
return self.protobuf.SerializeToString()
|
||||
|
||||
@property
|
||||
@@ -49,10 +52,9 @@ class ClaimDict(OrderedDict):
|
||||
|
||||
@property
|
||||
def has_signature(self):
|
||||
claim = self.protobuf
|
||||
if claim.HasField("publisherSignature"):
|
||||
return True
|
||||
return False
|
||||
return self.protobuf.HasField("publisherSignature") or (
|
||||
self.detached_signature and self.detached_signature.raw_signature
|
||||
)
|
||||
|
||||
@property
|
||||
def is_certificate(self):
|
||||
@@ -90,10 +92,11 @@ class ClaimDict(OrderedDict):
|
||||
return None
|
||||
|
||||
@property
|
||||
def certificate_id(self):
|
||||
if not self.has_signature:
|
||||
return None
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.certificateId)
|
||||
def certificate_id(self) -> str:
|
||||
if self.protobuf.HasField("publisherSignature"):
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.certificateId).decode()
|
||||
if self.detached_signature and self.detached_signature.certificate_id:
|
||||
return binascii.hexlify(self.detached_signature.certificate_id).decode()
|
||||
|
||||
@property
|
||||
def signature(self):
|
||||
@@ -117,57 +120,59 @@ class ClaimDict(OrderedDict):
|
||||
def claim_dict(self):
|
||||
"""Claim dictionary with bytes represented as hex and base58"""
|
||||
|
||||
return dict(encode_fields(self))
|
||||
return dict(encode_fields(self, self.detached_signature))
|
||||
|
||||
@classmethod
|
||||
def load_protobuf_dict(cls, protobuf_dict):
|
||||
def load_protobuf_dict(cls, protobuf_dict, detached_signature=None):
|
||||
"""
|
||||
Load a ClaimDict from a dictionary with base64 encoded bytes
|
||||
(as returned by the protobuf json formatter)
|
||||
"""
|
||||
|
||||
return cls(decode_b64_fields(protobuf_dict))
|
||||
return cls(decode_b64_fields(protobuf_dict), detached_signature=detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_protobuf(cls, protobuf_claim):
|
||||
def load_protobuf(cls, protobuf_claim, detached_signature=None):
|
||||
"""Load ClaimDict from a protobuf Claim message"""
|
||||
return cls.load_protobuf_dict(json.loads(json_format.MessageToJson(protobuf_claim, True)))
|
||||
return cls.load_protobuf_dict(json.loads(json_format.MessageToJson(protobuf_claim, True)), detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_dict(cls, claim_dict):
|
||||
"""Load ClaimDict from a dictionary with hex and base58 encoded bytes"""
|
||||
try:
|
||||
return cls.load_protobuf(cls(decode_fields(claim_dict)).protobuf)
|
||||
claim_dict, detached_signature = decode_fields(claim_dict)
|
||||
return cls.load_protobuf(cls(claim_dict).protobuf, detached_signature)
|
||||
except json_format.ParseError as err:
|
||||
raise DecodeError(str(err))
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, serialized):
|
||||
"""Load a ClaimDict from a serialized protobuf string"""
|
||||
detached_signature = Signature.flagged_parse(serialized)
|
||||
|
||||
temp_claim = claim_pb2.Claim()
|
||||
try:
|
||||
temp_claim.ParseFromString(serialized)
|
||||
temp_claim.ParseFromString(detached_signature.payload)
|
||||
except DecodeError_pb:
|
||||
raise DecodeError(DecodeError_pb)
|
||||
return cls.load_protobuf(temp_claim)
|
||||
return cls.load_protobuf(temp_claim, detached_signature=detached_signature)
|
||||
|
||||
@classmethod
|
||||
def generate_certificate(cls, private_key, curve=NIST256p):
|
||||
def generate_certificate(cls, private_key, curve=SECP256k1):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
return cls.load_protobuf(signer.certificate)
|
||||
|
||||
def sign(self, private_key, claim_address, cert_claim_id, curve=NIST256p):
|
||||
def sign(self, private_key, claim_address, cert_claim_id, curve=SECP256k1, name=None, legacy=False):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
signed = signer.sign_stream_claim(self, claim_address, cert_claim_id)
|
||||
return ClaimDict.load_protobuf(signed)
|
||||
signed, signature = signer.sign_stream_claim(self, claim_address, cert_claim_id, name, legacy)
|
||||
return ClaimDict.load_protobuf(signed, signature)
|
||||
|
||||
def validate_signature(self, claim_address, certificate):
|
||||
def validate_signature(self, claim_address, certificate, name=None):
|
||||
if isinstance(certificate, ClaimDict):
|
||||
certificate = certificate.protobuf
|
||||
curve = CURVE_NAMES[certificate.certificate.keyType]
|
||||
validator = get_validator(curve).load_from_certificate(certificate, self.certificate_id)
|
||||
return validator.validate_claim_signature(self, claim_address)
|
||||
return validator.validate_claim_signature(self, claim_address, name)
|
||||
|
||||
def validate_private_key(self, private_key, certificate_id):
|
||||
certificate = self.protobuf
|
||||
|
||||
@@ -4,15 +4,20 @@ from lbrynet.schema.address import decode_address, encode_address
|
||||
from lbrynet.schema.schema import CLAIM_TYPES, CLAIM_TYPE, STREAM_TYPE, CERTIFICATE_TYPE
|
||||
from lbrynet.schema.schema import SIGNATURE
|
||||
from lbrynet.schema.error import DecodeError, InvalidAddress
|
||||
from lbrynet.schema.signature import Signature
|
||||
|
||||
|
||||
def encode_fields(claim_dictionary):
|
||||
def encode_fields(claim_dictionary, detached_signature: Signature):
|
||||
"""Encode bytes to hex and b58 for return by ClaimDict"""
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
|
||||
claim_value['source']['source'] = binascii.hexlify(claim_value['source']['source']).decode()
|
||||
if 'releaseTime' in claim_value['metadata']:
|
||||
release_time = int(claim_value['metadata'].pop('releaseTime'))
|
||||
if release_time != 0:
|
||||
claim_value['metadata']['releaseTime'] = release_time
|
||||
if 'fee' in claim_value['metadata']:
|
||||
try:
|
||||
address = encode_address(claim_value['metadata']['fee']['address'])
|
||||
@@ -27,12 +32,19 @@ def encode_fields(claim_dictionary):
|
||||
encoded_cert_id = binascii.hexlify(claim_dictionary[SIGNATURE]['certificateId']).decode()
|
||||
claim_dictionary[SIGNATURE]['signature'] = encoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = encoded_cert_id
|
||||
elif detached_signature and detached_signature.raw_signature:
|
||||
claim_dictionary[SIGNATURE] = {
|
||||
'detached_signature': binascii.hexlify(detached_signature.serialized).decode(),
|
||||
'certificateId': binascii.hexlify(detached_signature.certificate_id).decode()
|
||||
}
|
||||
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
|
||||
|
||||
def decode_fields(claim_dictionary):
|
||||
"""Decode hex and b58 encoded bytes in dictionaries given to ClaimDict"""
|
||||
detached_signature = None
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
@@ -47,13 +59,17 @@ def decode_fields(claim_dictionary):
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = binascii.unhexlify(claim_value["publicKey"])
|
||||
claim_value["publicKey"] = public_key
|
||||
if SIGNATURE in claim_dictionary:
|
||||
if SIGNATURE in claim_dictionary and not claim_dictionary[SIGNATURE].get('detached_signature'):
|
||||
decoded_sig = binascii.unhexlify(claim_dictionary[SIGNATURE]['signature'])
|
||||
decoded_cert_id = binascii.unhexlify(claim_dictionary[SIGNATURE]['certificateId'])
|
||||
claim_dictionary[SIGNATURE]['signature'] = decoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = decoded_cert_id
|
||||
elif claim_dictionary.get(SIGNATURE, {}).get('detached_signature'):
|
||||
hex_detached_signature = claim_dictionary[SIGNATURE]['detached_signature']
|
||||
detached_signature = Signature.flagged_parse(binascii.unhexlify(hex_detached_signature))
|
||||
del claim_dictionary[SIGNATURE]
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
return claim_dictionary, detached_signature
|
||||
|
||||
|
||||
def decode_b64_fields(claim_dictionary):
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
enum KeyType {
|
||||
UNKNOWN_PUBLIC_KEY_TYPE = 0;
|
||||
NIST256p = 1;
|
||||
NIST384p = 2;
|
||||
SECP256k1 = 3;
|
||||
}
|
||||
|
||||
message Certificate {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required KeyType keyType = 2;
|
||||
required bytes publicKey = 4;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/certificate.proto
|
||||
# source: certificate.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -16,16 +16,16 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/certificate.proto',
|
||||
package='',
|
||||
name='certificate.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n&lbrynet/schema/proto/certificate.proto\"\x8e\x01\n\x0b\x43\x65rtificate\x12%\n\x07version\x18\x01 \x02(\x0e\x32\x14.Certificate.Version\x12\x19\n\x07keyType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
|
||||
serialized_pb=_b('\n\x11\x63\x65rtificate.proto\x12\x02pb\"\x94\x01\n\x0b\x43\x65rtificate\x12(\n\x07version\x18\x01 \x02(\x0e\x32\x17.pb.Certificate.Version\x12\x1c\n\x07keyType\x18\x02 \x02(\x0e\x32\x0b.pb.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
|
||||
)
|
||||
|
||||
_KEYTYPE = _descriptor.EnumDescriptor(
|
||||
name='KeyType',
|
||||
full_name='KeyType',
|
||||
full_name='pb.KeyType',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -48,8 +48,8 @@ _KEYTYPE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=187,
|
||||
serialized_end=268,
|
||||
serialized_start=176,
|
||||
serialized_end=257,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_KEYTYPE)
|
||||
|
||||
@@ -62,7 +62,7 @@ SECP256k1 = 3
|
||||
|
||||
_CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Certificate.Version',
|
||||
full_name='pb.Certificate.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -77,35 +77,35 @@ _CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=143,
|
||||
serialized_end=185,
|
||||
serialized_start=132,
|
||||
serialized_end=174,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CERTIFICATE_VERSION)
|
||||
|
||||
|
||||
_CERTIFICATE = _descriptor.Descriptor(
|
||||
name='Certificate',
|
||||
full_name='Certificate',
|
||||
full_name='pb.Certificate',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Certificate.version', index=0,
|
||||
name='version', full_name='pb.Certificate.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='keyType', full_name='Certificate.keyType', index=1,
|
||||
name='keyType', full_name='pb.Certificate.keyType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='publicKey', full_name='Certificate.publicKey', index=2,
|
||||
name='publicKey', full_name='pb.Certificate.publicKey', index=2,
|
||||
number=4, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -124,8 +124,8 @@ _CERTIFICATE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=43,
|
||||
serialized_end=185,
|
||||
serialized_start=26,
|
||||
serialized_end=174,
|
||||
)
|
||||
|
||||
_CERTIFICATE.fields_by_name['version'].enum_type = _CERTIFICATE_VERSION
|
||||
@@ -137,8 +137,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Certificate = _reflection.GeneratedProtocolMessageType('Certificate', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CERTIFICATE,
|
||||
__module__ = 'lbrynet.schema.proto.certificate_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Certificate)
|
||||
__module__ = 'certificate_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Certificate)
|
||||
))
|
||||
_sym_db.RegisterMessage(Certificate)
|
||||
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/stream.proto";
|
||||
import "lbrynet/schema/proto/certificate.proto";
|
||||
import "lbrynet/schema/proto/signature.proto";
|
||||
|
||||
message Claim {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
enum ClaimType {
|
||||
UNKNOWN_CLAIM_TYPE = 0;
|
||||
streamType = 1;
|
||||
certificateType = 2;
|
||||
}
|
||||
required ClaimType claimType = 2;
|
||||
optional Stream stream = 3;
|
||||
optional Certificate certificate = 4;
|
||||
optional Signature publisherSignature = 5;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/claim.proto
|
||||
# source: claim.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,25 +12,25 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import stream_pb2 as lbrynet_dot_schema_dot_proto_dot_stream__pb2
|
||||
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
|
||||
from lbrynet.schema.proto import signature_pb2 as lbrynet_dot_schema_dot_proto_dot_signature__pb2
|
||||
from . import stream_pb2 as stream__pb2
|
||||
from . import certificate_pb2 as certificate__pb2
|
||||
from . import signature_pb2 as signature__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/claim.proto',
|
||||
package='',
|
||||
name='claim.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n lbrynet/schema/proto/claim.proto\x1a!lbrynet/schema/proto/stream.proto\x1a&lbrynet/schema/proto/certificate.proto\x1a$lbrynet/schema/proto/signature.proto\"\xa7\x02\n\x05\x43laim\x12\x1f\n\x07version\x18\x01 \x02(\x0e\x32\x0e.Claim.Version\x12#\n\tclaimType\x18\x02 \x02(\x0e\x32\x10.Claim.ClaimType\x12\x17\n\x06stream\x18\x03 \x01(\x0b\x32\x07.Stream\x12!\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x0c.Certificate\x12&\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\n.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
|
||||
serialized_pb=_b('\n\x0b\x63laim.proto\x12\x02pb\x1a\x0cstream.proto\x1a\x11\x63\x65rtificate.proto\x1a\x0fsignature.proto\"\xb6\x02\n\x05\x43laim\x12\"\n\x07version\x18\x01 \x02(\x0e\x32\x11.pb.Claim.Version\x12&\n\tclaimType\x18\x02 \x02(\x0e\x32\x13.pb.Claim.ClaimType\x12\x1a\n\x06stream\x18\x03 \x01(\x0b\x32\n.pb.Stream\x12$\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x0f.pb.Certificate\x12)\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\r.pb.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_stream__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_signature__pb2.DESCRIPTOR,])
|
||||
dependencies=[stream__pb2.DESCRIPTOR,certificate__pb2.DESCRIPTOR,signature__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_CLAIM_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Claim.Version',
|
||||
full_name='pb.Claim.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -45,14 +45,14 @@ _CLAIM_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=329,
|
||||
serialized_end=371,
|
||||
serialized_start=264,
|
||||
serialized_end=306,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CLAIM_VERSION)
|
||||
|
||||
_CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
|
||||
name='ClaimType',
|
||||
full_name='Claim.ClaimType',
|
||||
full_name='pb.Claim.ClaimType',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -71,49 +71,49 @@ _CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=373,
|
||||
serialized_end=445,
|
||||
serialized_start=308,
|
||||
serialized_end=380,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_CLAIM_CLAIMTYPE)
|
||||
|
||||
|
||||
_CLAIM = _descriptor.Descriptor(
|
||||
name='Claim',
|
||||
full_name='Claim',
|
||||
full_name='pb.Claim',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Claim.version', index=0,
|
||||
name='version', full_name='pb.Claim.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='claimType', full_name='Claim.claimType', index=1,
|
||||
name='claimType', full_name='pb.Claim.claimType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='stream', full_name='Claim.stream', index=2,
|
||||
name='stream', full_name='pb.Claim.stream', index=2,
|
||||
number=3, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='certificate', full_name='Claim.certificate', index=3,
|
||||
name='certificate', full_name='pb.Claim.certificate', index=3,
|
||||
number=4, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='publisherSignature', full_name='Claim.publisherSignature', index=4,
|
||||
name='publisherSignature', full_name='pb.Claim.publisherSignature', index=4,
|
||||
number=5, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -133,15 +133,15 @@ _CLAIM = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=150,
|
||||
serialized_end=445,
|
||||
serialized_start=70,
|
||||
serialized_end=380,
|
||||
)
|
||||
|
||||
_CLAIM.fields_by_name['version'].enum_type = _CLAIM_VERSION
|
||||
_CLAIM.fields_by_name['claimType'].enum_type = _CLAIM_CLAIMTYPE
|
||||
_CLAIM.fields_by_name['stream'].message_type = lbrynet_dot_schema_dot_proto_dot_stream__pb2._STREAM
|
||||
_CLAIM.fields_by_name['certificate'].message_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._CERTIFICATE
|
||||
_CLAIM.fields_by_name['publisherSignature'].message_type = lbrynet_dot_schema_dot_proto_dot_signature__pb2._SIGNATURE
|
||||
_CLAIM.fields_by_name['stream'].message_type = stream__pb2._STREAM
|
||||
_CLAIM.fields_by_name['certificate'].message_type = certificate__pb2._CERTIFICATE
|
||||
_CLAIM.fields_by_name['publisherSignature'].message_type = signature__pb2._SIGNATURE
|
||||
_CLAIM_VERSION.containing_type = _CLAIM
|
||||
_CLAIM_CLAIMTYPE.containing_type = _CLAIM
|
||||
DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
|
||||
@@ -149,8 +149,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict(
|
||||
DESCRIPTOR = _CLAIM,
|
||||
__module__ = 'lbrynet.schema.proto.claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Claim)
|
||||
__module__ = 'claim_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Claim)
|
||||
))
|
||||
_sym_db.RegisterMessage(Claim)
|
||||
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
message Fee {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
enum Currency {
|
||||
UNKNOWN_CURRENCY = 0;
|
||||
LBC = 1;
|
||||
BTC = 2;
|
||||
USD = 3;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Currency currency = 2;
|
||||
required bytes address = 3;
|
||||
required float amount = 4;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/fee.proto
|
||||
# source: fee.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/fee.proto',
|
||||
package='',
|
||||
name='fee.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n\x1elbrynet/schema/proto/fee.proto\"\xcf\x01\n\x03\x46\x65\x65\x12\x1d\n\x07version\x18\x01 \x02(\x0e\x32\x0c.Fee.Version\x12\x1f\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\r.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
|
||||
serialized_pb=_b('\n\tfee.proto\x12\x02pb\"\xd5\x01\n\x03\x46\x65\x65\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.pb.Fee.Version\x12\"\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\x10.pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
|
||||
)
|
||||
|
||||
|
||||
|
||||
_FEE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Fee.Version',
|
||||
full_name='pb.Fee.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -41,14 +41,14 @@ _FEE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=139,
|
||||
serialized_end=181,
|
||||
serialized_start=128,
|
||||
serialized_end=170,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_FEE_VERSION)
|
||||
|
||||
_FEE_CURRENCY = _descriptor.EnumDescriptor(
|
||||
name='Currency',
|
||||
full_name='Fee.Currency',
|
||||
full_name='pb.Fee.Currency',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -71,42 +71,42 @@ _FEE_CURRENCY = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=183,
|
||||
serialized_end=242,
|
||||
serialized_start=172,
|
||||
serialized_end=231,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
|
||||
|
||||
|
||||
_FEE = _descriptor.Descriptor(
|
||||
name='Fee',
|
||||
full_name='Fee',
|
||||
full_name='pb.Fee',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Fee.version', index=0,
|
||||
name='version', full_name='pb.Fee.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='currency', full_name='Fee.currency', index=1,
|
||||
name='currency', full_name='pb.Fee.currency', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='address', full_name='Fee.address', index=2,
|
||||
name='address', full_name='pb.Fee.address', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='amount', full_name='Fee.amount', index=3,
|
||||
name='amount', full_name='pb.Fee.amount', index=3,
|
||||
number=4, type=2, cpp_type=6, label=2,
|
||||
has_default_value=False, default_value=float(0),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -126,8 +126,8 @@ _FEE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=35,
|
||||
serialized_end=242,
|
||||
serialized_start=18,
|
||||
serialized_end=231,
|
||||
)
|
||||
|
||||
_FEE.fields_by_name['version'].enum_type = _FEE_VERSION
|
||||
@@ -139,8 +139,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
|
||||
DESCRIPTOR = _FEE,
|
||||
__module__ = 'lbrynet.schema.proto.fee_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Fee)
|
||||
__module__ = 'fee_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Fee)
|
||||
))
|
||||
_sym_db.RegisterMessage(Fee)
|
||||
|
||||
|
||||
@@ -1,212 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/fee.proto";
|
||||
|
||||
message Metadata {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
_0_0_2 = 2;
|
||||
_0_0_3 = 3;
|
||||
_0_1_0 = 4;
|
||||
}
|
||||
enum Language {
|
||||
UNKNOWN_LANGUAGE = 0;
|
||||
en = 1;
|
||||
aa = 2;
|
||||
ab = 3;
|
||||
ae = 4;
|
||||
af = 5;
|
||||
ak = 6;
|
||||
am = 7;
|
||||
an = 8;
|
||||
ar = 9;
|
||||
as = 10;
|
||||
av = 11;
|
||||
ay = 12;
|
||||
az = 13;
|
||||
ba = 14;
|
||||
be = 15;
|
||||
bg = 16;
|
||||
bh = 17;
|
||||
bi = 18;
|
||||
bm = 19;
|
||||
bn = 20;
|
||||
bo = 21;
|
||||
br = 22;
|
||||
bs = 23;
|
||||
ca = 24;
|
||||
ce = 25;
|
||||
ch = 26;
|
||||
co = 27;
|
||||
cr = 28;
|
||||
cs = 29;
|
||||
cu = 30;
|
||||
cv = 31;
|
||||
cy = 32;
|
||||
da = 33;
|
||||
de = 34;
|
||||
dv = 35;
|
||||
dz = 36;
|
||||
ee = 37;
|
||||
el = 38;
|
||||
eo = 39;
|
||||
es = 40;
|
||||
et = 41;
|
||||
eu = 42;
|
||||
fa = 43;
|
||||
ff = 44;
|
||||
fi = 45;
|
||||
fj = 46;
|
||||
fo = 47;
|
||||
fr = 48;
|
||||
fy = 49;
|
||||
ga = 50;
|
||||
gd = 51;
|
||||
gl = 52;
|
||||
gn = 53;
|
||||
gu = 54;
|
||||
gv = 55;
|
||||
ha = 56;
|
||||
he = 57;
|
||||
hi = 58;
|
||||
ho = 59;
|
||||
hr = 60;
|
||||
ht = 61;
|
||||
hu = 62;
|
||||
hy = 63;
|
||||
hz = 64;
|
||||
ia = 65;
|
||||
id = 66;
|
||||
ie = 67;
|
||||
ig = 68;
|
||||
ii = 69;
|
||||
ik = 70;
|
||||
io = 71;
|
||||
is = 72;
|
||||
it = 73;
|
||||
iu = 74;
|
||||
ja = 75;
|
||||
jv = 76;
|
||||
ka = 77;
|
||||
kg = 78;
|
||||
ki = 79;
|
||||
kj = 80;
|
||||
kk = 81;
|
||||
kl = 82;
|
||||
km = 83;
|
||||
kn = 84;
|
||||
ko = 85;
|
||||
kr = 86;
|
||||
ks = 87;
|
||||
ku = 88;
|
||||
kv = 89;
|
||||
kw = 90;
|
||||
ky = 91;
|
||||
la = 92;
|
||||
lb = 93;
|
||||
lg = 94;
|
||||
li = 95;
|
||||
ln = 96;
|
||||
lo = 97;
|
||||
lt = 98;
|
||||
lu = 99;
|
||||
lv = 100;
|
||||
mg = 101;
|
||||
mh = 102;
|
||||
mi = 103;
|
||||
mk = 104;
|
||||
ml = 105;
|
||||
mn = 106;
|
||||
mr = 107;
|
||||
ms = 108;
|
||||
mt = 109;
|
||||
my = 110;
|
||||
na = 111;
|
||||
nb = 112;
|
||||
nd = 113;
|
||||
ne = 114;
|
||||
ng = 115;
|
||||
nl = 116;
|
||||
nn = 117;
|
||||
no = 118;
|
||||
nr = 119;
|
||||
nv = 120;
|
||||
ny = 121;
|
||||
oc = 122;
|
||||
oj = 123;
|
||||
om = 124;
|
||||
or = 125;
|
||||
os = 126;
|
||||
pa = 127;
|
||||
pi = 128;
|
||||
pl = 129;
|
||||
ps = 130;
|
||||
pt = 131;
|
||||
qu = 132;
|
||||
rm = 133;
|
||||
rn = 134;
|
||||
ro = 135;
|
||||
ru = 136;
|
||||
rw = 137;
|
||||
sa = 138;
|
||||
sc = 139;
|
||||
sd = 140;
|
||||
se = 141;
|
||||
sg = 142;
|
||||
si = 143;
|
||||
sk = 144;
|
||||
sl = 145;
|
||||
sm = 146;
|
||||
sn = 147;
|
||||
so = 148;
|
||||
sq = 149;
|
||||
sr = 150;
|
||||
ss = 151;
|
||||
st = 152;
|
||||
su = 153;
|
||||
sv = 154;
|
||||
sw = 155;
|
||||
ta = 156;
|
||||
te = 157;
|
||||
tg = 158;
|
||||
th = 159;
|
||||
ti = 160;
|
||||
tk = 161;
|
||||
tl = 162;
|
||||
tn = 163;
|
||||
to = 164;
|
||||
tr = 165;
|
||||
ts = 166;
|
||||
tt = 167;
|
||||
tw = 168;
|
||||
ty = 169;
|
||||
ug = 170;
|
||||
uk = 171;
|
||||
ur = 172;
|
||||
uz = 173;
|
||||
ve = 174;
|
||||
vi = 175;
|
||||
vo = 176;
|
||||
wa = 177;
|
||||
wo = 178;
|
||||
xh = 179;
|
||||
yi = 180;
|
||||
yo = 181;
|
||||
za = 182;
|
||||
zh = 183;
|
||||
zu = 184;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Language language = 2;
|
||||
required string title = 3;
|
||||
required string description = 4;
|
||||
required string author = 5;
|
||||
required string license = 6;
|
||||
required bool nsfw = 7;
|
||||
|
||||
optional Fee fee = 8;
|
||||
optional string thumbnail = 9;
|
||||
optional string preview = 10;
|
||||
optional string licenseUrl = 11;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/metadata.proto
|
||||
# source: metadata.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import fee_pb2 as lbrynet_dot_schema_dot_proto_dot_fee__pb2
|
||||
from . import fee_pb2 as fee__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/metadata.proto',
|
||||
package='',
|
||||
name='metadata.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n#lbrynet/schema/proto/metadata.proto\x1a\x1elbrynet/schema/proto/fee.proto\"\xde\x0e\n\x08Metadata\x12\"\n\x07version\x18\x01 \x02(\x0e\x32\x11.Metadata.Version\x12$\n\x08language\x18\x02 \x02(\x0e\x32\x12.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x11\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x04.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02ty\x1Line truncated
|
||||
serialized_pb=_b('\n\x0emetadata.proto\x12\x02pb\x1a\tfee.proto\"\xfc\x0e\n\x08Metadata\x12%\n\x07version\x18\x01 \x02(\x0e\x32\x14.pb.Metadata.Version\x12\'\n\x08language\x18\x02 \x02(\x0e\x32\x15.pb.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x14\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x07.pb.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\x12\x13\n\x0breleaseTime\x18\x0c \x01(\x12\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\xLine truncated
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_fee__pb2.DESCRIPTOR,])
|
||||
dependencies=[fee__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_METADATA_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Metadata.Version',
|
||||
full_name='pb.Metadata.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -55,14 +55,14 @@ _METADATA_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=316,
|
||||
serialized_end=394,
|
||||
serialized_start=308,
|
||||
serialized_end=386,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_METADATA_VERSION)
|
||||
|
||||
_METADATA_LANGUAGE = _descriptor.EnumDescriptor(
|
||||
name='Language',
|
||||
full_name='Metadata.Language',
|
||||
full_name='pb.Metadata.Language',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -809,96 +809,103 @@ _METADATA_LANGUAGE = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=397,
|
||||
serialized_end=1958,
|
||||
serialized_start=389,
|
||||
serialized_end=1950,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_METADATA_LANGUAGE)
|
||||
|
||||
|
||||
_METADATA = _descriptor.Descriptor(
|
||||
name='Metadata',
|
||||
full_name='Metadata',
|
||||
full_name='pb.Metadata',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Metadata.version', index=0,
|
||||
name='version', full_name='pb.Metadata.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='language', full_name='Metadata.language', index=1,
|
||||
name='language', full_name='pb.Metadata.language', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='title', full_name='Metadata.title', index=2,
|
||||
name='title', full_name='pb.Metadata.title', index=2,
|
||||
number=3, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='description', full_name='Metadata.description', index=3,
|
||||
name='description', full_name='pb.Metadata.description', index=3,
|
||||
number=4, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='author', full_name='Metadata.author', index=4,
|
||||
name='author', full_name='pb.Metadata.author', index=4,
|
||||
number=5, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='license', full_name='Metadata.license', index=5,
|
||||
name='license', full_name='pb.Metadata.license', index=5,
|
||||
number=6, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='nsfw', full_name='Metadata.nsfw', index=6,
|
||||
name='nsfw', full_name='pb.Metadata.nsfw', index=6,
|
||||
number=7, type=8, cpp_type=7, label=2,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='fee', full_name='Metadata.fee', index=7,
|
||||
name='fee', full_name='pb.Metadata.fee', index=7,
|
||||
number=8, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='thumbnail', full_name='Metadata.thumbnail', index=8,
|
||||
name='thumbnail', full_name='pb.Metadata.thumbnail', index=8,
|
||||
number=9, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='preview', full_name='Metadata.preview', index=9,
|
||||
name='preview', full_name='pb.Metadata.preview', index=9,
|
||||
number=10, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='licenseUrl', full_name='Metadata.licenseUrl', index=10,
|
||||
name='licenseUrl', full_name='pb.Metadata.licenseUrl', index=10,
|
||||
number=11, type=9, cpp_type=9, label=1,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='releaseTime', full_name='pb.Metadata.releaseTime', index=11,
|
||||
number=12, type=18, cpp_type=2, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
@@ -913,13 +920,13 @@ _METADATA = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=72,
|
||||
serialized_end=1958,
|
||||
serialized_start=34,
|
||||
serialized_end=1950,
|
||||
)
|
||||
|
||||
_METADATA.fields_by_name['version'].enum_type = _METADATA_VERSION
|
||||
_METADATA.fields_by_name['language'].enum_type = _METADATA_LANGUAGE
|
||||
_METADATA.fields_by_name['fee'].message_type = lbrynet_dot_schema_dot_proto_dot_fee__pb2._FEE
|
||||
_METADATA.fields_by_name['fee'].message_type = fee__pb2._FEE
|
||||
_METADATA_VERSION.containing_type = _METADATA
|
||||
_METADATA_LANGUAGE.containing_type = _METADATA
|
||||
DESCRIPTOR.message_types_by_name['Metadata'] = _METADATA
|
||||
@@ -927,8 +934,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Metadata = _reflection.GeneratedProtocolMessageType('Metadata', (_message.Message,), dict(
|
||||
DESCRIPTOR = _METADATA,
|
||||
__module__ = 'lbrynet.schema.proto.metadata_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Metadata)
|
||||
__module__ = 'metadata_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Metadata)
|
||||
))
|
||||
_sym_db.RegisterMessage(Metadata)
|
||||
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/certificate.proto";
|
||||
|
||||
message Signature {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required KeyType signatureType = 2;
|
||||
required bytes signature = 3;
|
||||
required bytes certificateId = 4;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/signature.proto
|
||||
# source: signature.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
|
||||
from . import certificate_pb2 as certificate__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/signature.proto',
|
||||
package='',
|
||||
name='signature.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n$lbrynet/schema/proto/signature.proto\x1a&lbrynet/schema/proto/certificate.proto\"\xa7\x01\n\tSignature\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.Signature.Version\x12\x1f\n\rsignatureType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
serialized_pb=_b('\n\x0fsignature.proto\x12\x02pb\x1a\x11\x63\x65rtificate.proto\"\xad\x01\n\tSignature\x12&\n\x07version\x18\x01 \x02(\x0e\x32\x15.pb.Signature.Version\x12\"\n\rsignatureType\x18\x02 \x02(\x0e\x32\x0b.pb.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,])
|
||||
dependencies=[certificate__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_SIGNATURE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Signature.Version',
|
||||
full_name='pb.Signature.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -43,42 +43,42 @@ _SIGNATURE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=206,
|
||||
serialized_end=248,
|
||||
serialized_start=174,
|
||||
serialized_end=216,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SIGNATURE_VERSION)
|
||||
|
||||
|
||||
_SIGNATURE = _descriptor.Descriptor(
|
||||
name='Signature',
|
||||
full_name='Signature',
|
||||
full_name='pb.Signature',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Signature.version', index=0,
|
||||
name='version', full_name='pb.Signature.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signatureType', full_name='Signature.signatureType', index=1,
|
||||
name='signatureType', full_name='pb.Signature.signatureType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='signature', full_name='Signature.signature', index=2,
|
||||
name='signature', full_name='pb.Signature.signature', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='certificateId', full_name='Signature.certificateId', index=3,
|
||||
name='certificateId', full_name='pb.Signature.certificateId', index=3,
|
||||
number=4, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -97,20 +97,20 @@ _SIGNATURE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=81,
|
||||
serialized_end=248,
|
||||
serialized_start=43,
|
||||
serialized_end=216,
|
||||
)
|
||||
|
||||
_SIGNATURE.fields_by_name['version'].enum_type = _SIGNATURE_VERSION
|
||||
_SIGNATURE.fields_by_name['signatureType'].enum_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._KEYTYPE
|
||||
_SIGNATURE.fields_by_name['signatureType'].enum_type = certificate__pb2._KEYTYPE
|
||||
_SIGNATURE_VERSION.containing_type = _SIGNATURE
|
||||
DESCRIPTOR.message_types_by_name['Signature'] = _SIGNATURE
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Signature = _reflection.GeneratedProtocolMessageType('Signature', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SIGNATURE,
|
||||
__module__ = 'lbrynet.schema.proto.signature_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Signature)
|
||||
__module__ = 'signature_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Signature)
|
||||
))
|
||||
_sym_db.RegisterMessage(Signature)
|
||||
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
message Source {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
enum SourceTypes {
|
||||
UNKNOWN_SOURCE_TYPE = 0;
|
||||
lbry_sd_hash = 1;
|
||||
}
|
||||
required SourceTypes sourceType = 2;
|
||||
required bytes source = 3;
|
||||
required string contentType = 4;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/source.proto
|
||||
# source: source.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/source.proto',
|
||||
package='',
|
||||
name='source.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n!lbrynet/schema/proto/source.proto\"\xde\x01\n\x06Source\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Source.Version\x12\'\n\nsourceType\x18\x02 \x02(\x0e\x32\x13.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
|
||||
serialized_pb=_b('\n\x0csource.proto\x12\x02pb\"\xe4\x01\n\x06Source\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.pb.Source.Version\x12*\n\nsourceType\x18\x02 \x02(\x0e\x32\x16.pb.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
|
||||
)
|
||||
|
||||
|
||||
|
||||
_SOURCE_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Source.Version',
|
||||
full_name='pb.Source.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -41,14 +41,14 @@ _SOURCE_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=160,
|
||||
serialized_end=202,
|
||||
serialized_start=149,
|
||||
serialized_end=191,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SOURCE_VERSION)
|
||||
|
||||
_SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
|
||||
name='SourceTypes',
|
||||
full_name='Source.SourceTypes',
|
||||
full_name='pb.Source.SourceTypes',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -63,42 +63,42 @@ _SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=204,
|
||||
serialized_end=260,
|
||||
serialized_start=193,
|
||||
serialized_end=249,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_SOURCE_SOURCETYPES)
|
||||
|
||||
|
||||
_SOURCE = _descriptor.Descriptor(
|
||||
name='Source',
|
||||
full_name='Source',
|
||||
full_name='pb.Source',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Source.version', index=0,
|
||||
name='version', full_name='pb.Source.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='sourceType', full_name='Source.sourceType', index=1,
|
||||
name='sourceType', full_name='pb.Source.sourceType', index=1,
|
||||
number=2, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='source', full_name='Source.source', index=2,
|
||||
name='source', full_name='pb.Source.source', index=2,
|
||||
number=3, type=12, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b(""),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='contentType', full_name='Source.contentType', index=3,
|
||||
name='contentType', full_name='pb.Source.contentType', index=3,
|
||||
number=4, type=9, cpp_type=9, label=2,
|
||||
has_default_value=False, default_value=_b("").decode('utf-8'),
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -118,8 +118,8 @@ _SOURCE = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=38,
|
||||
serialized_end=260,
|
||||
serialized_start=21,
|
||||
serialized_end=249,
|
||||
)
|
||||
|
||||
_SOURCE.fields_by_name['version'].enum_type = _SOURCE_VERSION
|
||||
@@ -131,8 +131,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
|
||||
DESCRIPTOR = _SOURCE,
|
||||
__module__ = 'lbrynet.schema.proto.source_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Source)
|
||||
__module__ = 'source_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Source)
|
||||
))
|
||||
_sym_db.RegisterMessage(Source)
|
||||
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
syntax = "proto2";
|
||||
|
||||
import "lbrynet/schema/proto/metadata.proto";
|
||||
import "lbrynet/schema/proto/source.proto";
|
||||
|
||||
|
||||
message Stream {
|
||||
enum Version {
|
||||
UNKNOWN_VERSION = 0;
|
||||
_0_0_1 = 1;
|
||||
}
|
||||
required Version version = 1;
|
||||
required Metadata metadata = 2;
|
||||
required Source source = 3;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: lbrynet/schema/proto/stream.proto
|
||||
# source: stream.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
@@ -12,24 +12,24 @@ from google.protobuf import symbol_database as _symbol_database
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
from lbrynet.schema.proto import metadata_pb2 as lbrynet_dot_schema_dot_proto_dot_metadata__pb2
|
||||
from lbrynet.schema.proto import source_pb2 as lbrynet_dot_schema_dot_proto_dot_source__pb2
|
||||
from . import metadata_pb2 as metadata__pb2
|
||||
from . import source_pb2 as source__pb2
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='lbrynet/schema/proto/stream.proto',
|
||||
package='',
|
||||
name='stream.proto',
|
||||
package='pb',
|
||||
syntax='proto2',
|
||||
serialized_options=None,
|
||||
serialized_pb=_b('\n!lbrynet/schema/proto/stream.proto\x1a#lbrynet/schema/proto/metadata.proto\x1a!lbrynet/schema/proto/source.proto\"\x8c\x01\n\x06Stream\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Stream.Version\x12\x1b\n\x08metadata\x18\x02 \x02(\x0b\x32\t.Metadata\x12\x17\n\x06source\x18\x03 \x02(\x0b\x32\x07.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
serialized_pb=_b('\n\x0cstream.proto\x12\x02pb\x1a\x0emetadata.proto\x1a\x0csource.proto\"\x95\x01\n\x06Stream\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.pb.Stream.Version\x12\x1e\n\x08metadata\x18\x02 \x02(\x0b\x32\x0c.pb.Metadata\x12\x1a\n\x06source\x18\x03 \x02(\x0b\x32\n.pb.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
|
||||
,
|
||||
dependencies=[lbrynet_dot_schema_dot_proto_dot_metadata__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_source__pb2.DESCRIPTOR,])
|
||||
dependencies=[metadata__pb2.DESCRIPTOR,source__pb2.DESCRIPTOR,])
|
||||
|
||||
|
||||
|
||||
_STREAM_VERSION = _descriptor.EnumDescriptor(
|
||||
name='Version',
|
||||
full_name='Stream.Version',
|
||||
full_name='pb.Stream.Version',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
values=[
|
||||
@@ -44,35 +44,35 @@ _STREAM_VERSION = _descriptor.EnumDescriptor(
|
||||
],
|
||||
containing_type=None,
|
||||
serialized_options=None,
|
||||
serialized_start=208,
|
||||
serialized_end=250,
|
||||
serialized_start=158,
|
||||
serialized_end=200,
|
||||
)
|
||||
_sym_db.RegisterEnumDescriptor(_STREAM_VERSION)
|
||||
|
||||
|
||||
_STREAM = _descriptor.Descriptor(
|
||||
name='Stream',
|
||||
full_name='Stream',
|
||||
full_name='pb.Stream',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='version', full_name='Stream.version', index=0,
|
||||
name='version', full_name='pb.Stream.version', index=0,
|
||||
number=1, type=14, cpp_type=8, label=2,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='metadata', full_name='Stream.metadata', index=1,
|
||||
name='metadata', full_name='pb.Stream.metadata', index=1,
|
||||
number=2, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
serialized_options=None, file=DESCRIPTOR),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='source', full_name='Stream.source', index=2,
|
||||
name='source', full_name='pb.Stream.source', index=2,
|
||||
number=3, type=11, cpp_type=10, label=2,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
@@ -91,21 +91,21 @@ _STREAM = _descriptor.Descriptor(
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=110,
|
||||
serialized_end=250,
|
||||
serialized_start=51,
|
||||
serialized_end=200,
|
||||
)
|
||||
|
||||
_STREAM.fields_by_name['version'].enum_type = _STREAM_VERSION
|
||||
_STREAM.fields_by_name['metadata'].message_type = lbrynet_dot_schema_dot_proto_dot_metadata__pb2._METADATA
|
||||
_STREAM.fields_by_name['source'].message_type = lbrynet_dot_schema_dot_proto_dot_source__pb2._SOURCE
|
||||
_STREAM.fields_by_name['metadata'].message_type = metadata__pb2._METADATA
|
||||
_STREAM.fields_by_name['source'].message_type = source__pb2._SOURCE
|
||||
_STREAM_VERSION.containing_type = _STREAM
|
||||
DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
|
||||
DESCRIPTOR = _STREAM,
|
||||
__module__ = 'lbrynet.schema.proto.stream_pb2'
|
||||
# @@protoc_insertion_point(class_scope:Stream)
|
||||
__module__ = 'stream_pb2'
|
||||
# @@protoc_insertion_point(class_scope:pb.Stream)
|
||||
))
|
||||
_sym_db.RegisterMessage(Stream)
|
||||
|
||||
|
||||
@@ -14,4 +14,8 @@ class Metadata(Schema):
|
||||
if 'fee' in _metadata:
|
||||
fee_pb = Fee.load(_metadata.pop('fee'))
|
||||
_message_pb.fee.CopyFrom(fee_pb)
|
||||
return cls._load(_metadata, _message_pb)
|
||||
_message_pb.releaseTime = int(_metadata.get('releaseTime', 0))
|
||||
built_message = cls._load(_metadata, _message_pb)
|
||||
if built_message.releaseTime == 0:
|
||||
built_message.ClearField('releaseTime')
|
||||
return built_message
|
||||
@@ -0,0 +1,44 @@
|
||||
from collections import namedtuple
|
||||
|
||||
from typing import Union
|
||||
|
||||
LEGACY = namedtuple('Legacy', 'payload')
|
||||
NAMED_SECP256K1 = namedtuple('NamedSECP256k1', 'raw_signature certificate_id payload')
|
||||
FLAGS = {
|
||||
LEGACY: 0x80,
|
||||
NAMED_SECP256K1: 0x01
|
||||
}
|
||||
|
||||
class Signature:
|
||||
|
||||
def __init__(self, data: Union[LEGACY, NAMED_SECP256K1]):
|
||||
self.data = data
|
||||
|
||||
@property
|
||||
def payload(self):
|
||||
return self.data.payload
|
||||
|
||||
@property
|
||||
def certificate_id(self):
|
||||
if type(self.data) == NAMED_SECP256K1:
|
||||
return self.data.certificate_id
|
||||
|
||||
@property
|
||||
def raw_signature(self):
|
||||
if type(self.data) == NAMED_SECP256K1:
|
||||
return self.data.raw_signature
|
||||
|
||||
@classmethod
|
||||
def flagged_parse(cls, binary: bytes):
|
||||
flag = binary[0]
|
||||
if flag == FLAGS[NAMED_SECP256K1]:
|
||||
return cls(NAMED_SECP256K1(binary[1:65], binary[65:85], binary[85:]))
|
||||
else:
|
||||
return cls(LEGACY(binary))
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
if isinstance(self.data, NAMED_SECP256K1):
|
||||
return (bytes([FLAGS[type(self.data)]]) + self.data.raw_signature + self.data.certificate_id + self.payload)
|
||||
elif isinstance(self.data, LEGACY):
|
||||
return self.payload
|
||||
+29
-10
@@ -5,6 +5,7 @@ from lbrynet.schema.address import decode_address
|
||||
from lbrynet.schema.encoding import decode_b64_fields
|
||||
from lbrynet.schema.schema.certificate import Certificate
|
||||
from lbrynet.schema.schema.claim import Claim
|
||||
from lbrynet.schema.signature import Signature, NAMED_SECP256K1
|
||||
from lbrynet.schema.validator import validate_claim_id
|
||||
from lbrynet.schema.schema import V_0_0_1, CLAIM_TYPE, CLAIM_TYPES, CERTIFICATE_TYPE, VERSION
|
||||
from lbrynet.schema.schema import NIST256p, NIST384p, SECP256k1, SHA256, SHA384
|
||||
@@ -45,24 +46,41 @@ class NIST_ECDSASigner(object):
|
||||
def generate(cls):
|
||||
return cls(ecdsa.SigningKey.generate(curve=cls.CURVE, hashfunc=cls.HASHFUNC_NAME))
|
||||
|
||||
def sign_stream_claim(self, claim, claim_address, cert_claim_id):
|
||||
def sign(self, *fields):
|
||||
digest = self.HASHFUNC(bytearray(b''.join(fields))).digest()
|
||||
return self.private_key.sign_digest_deterministic(digest, hashfunc=self.HASHFUNC)
|
||||
|
||||
def sign_stream_claim(self, claim, claim_address, cert_claim_id, name, legacy=False):
|
||||
validate_claim_id(cert_claim_id)
|
||||
raw_cert_id = binascii.unhexlify(cert_claim_id)
|
||||
decoded_addr = decode_address(claim_address)
|
||||
if not legacy:
|
||||
assert name, "Name is required for detached signatures"
|
||||
assert self.CURVE_NAME == SECP256k1, f"Only SECP256k1 is supported, not: {self.CURVE_NAME}"
|
||||
signature = self.sign(
|
||||
name.lower().encode(),
|
||||
decoded_addr,
|
||||
claim.serialized_no_signature,
|
||||
raw_cert_id,
|
||||
)
|
||||
else:
|
||||
signature = self.sign(decoded_addr, claim.serialized_no_signature, raw_cert_id)
|
||||
|
||||
to_sign = bytearray()
|
||||
to_sign.extend(decoded_addr)
|
||||
to_sign.extend(claim.serialized_no_signature)
|
||||
to_sign.extend(binascii.unhexlify(cert_claim_id))
|
||||
|
||||
digest = self.HASHFUNC(to_sign).digest()
|
||||
if not legacy:
|
||||
return Claim.load(decode_b64_fields(claim.protobuf_dict)), Signature(NAMED_SECP256K1(
|
||||
signature,
|
||||
raw_cert_id,
|
||||
claim.serialized_no_signature
|
||||
))
|
||||
# -- Legacy signer (signature inside protobuf) --
|
||||
|
||||
if not isinstance(self.private_key, ecdsa.SigningKey):
|
||||
raise Exception("Not given a signing key")
|
||||
sig_dict = {
|
||||
"version": V_0_0_1,
|
||||
"signatureType": self.CURVE_NAME,
|
||||
"signature": self.private_key.sign_digest_deterministic(digest, hashfunc=self.HASHFUNC),
|
||||
"certificateId": binascii.unhexlify(cert_claim_id)
|
||||
"signature": signature,
|
||||
"certificateId": raw_cert_id
|
||||
}
|
||||
|
||||
msg = {
|
||||
@@ -71,7 +89,8 @@ class NIST_ECDSASigner(object):
|
||||
"publisherSignature": sig_dict
|
||||
}
|
||||
|
||||
return Claim.load(msg)
|
||||
proto = Claim.load(msg)
|
||||
return proto, Signature.flagged_parse(proto.SerializeToString())
|
||||
|
||||
|
||||
class NIST256pSigner(NIST_ECDSASigner):
|
||||
|
||||
+13
-2
@@ -28,7 +28,7 @@ class URI(object):
|
||||
self.claim_id = claim_id
|
||||
self.path = path
|
||||
|
||||
if self.path is not None and not self.is_channel:
|
||||
if self.path is not None and not self.contains_channel:
|
||||
raise ValueError("Content claims cannot have paths")
|
||||
|
||||
def __str__(self):
|
||||
@@ -39,10 +39,21 @@ class URI(object):
|
||||
if not hasattr(other, prop) or getattr(self, prop) != getattr(other, prop):
|
||||
return False
|
||||
return self.__class__ == other.__class__
|
||||
@property
|
||||
def channel_name(self):
|
||||
return self.name if self.contains_channel else None
|
||||
|
||||
@property
|
||||
def claim_name(self):
|
||||
return self.name if not self.contains_channel else self.path
|
||||
|
||||
@property
|
||||
def contains_channel(self):
|
||||
return self.name.startswith(CHANNEL_CHAR)
|
||||
|
||||
@property
|
||||
def is_channel(self):
|
||||
return self.name.startswith(CHANNEL_CHAR)
|
||||
return self.contains_channel and not self.path
|
||||
|
||||
def to_uri_string(self):
|
||||
uri_string = PROTOCOL + "%s" % self.name
|
||||
|
||||
@@ -76,18 +76,25 @@ class Validator:
|
||||
from ecdsa import BadSignatureError
|
||||
raise BadSignatureError
|
||||
|
||||
def validate_claim_signature(self, claim, claim_address):
|
||||
def validate_claim_signature(self, claim, claim_address, name):
|
||||
to_sign = bytearray()
|
||||
if claim.detached_signature and claim.detached_signature.raw_signature:
|
||||
assert name is not None, "Name is required for verifying detached signatures."
|
||||
to_sign.extend(name.lower().encode())
|
||||
signature = claim.detached_signature.raw_signature
|
||||
payload = claim.detached_signature.payload
|
||||
else:
|
||||
# extract and serialize the stream from the claim, then check the signature
|
||||
signature = binascii.unhexlify(claim.signature)
|
||||
payload = claim.serialized_no_signature
|
||||
decoded_address = decode_address(claim_address)
|
||||
|
||||
# extract and serialize the stream from the claim, then check the signature
|
||||
signature = binascii.unhexlify(claim.signature)
|
||||
|
||||
if signature is None:
|
||||
raise Exception("No signature to validate")
|
||||
|
||||
to_sign = bytearray()
|
||||
to_sign.extend(decoded_address)
|
||||
to_sign.extend(claim.serialized_no_signature)
|
||||
to_sign.extend(payload)
|
||||
to_sign.extend(binascii.unhexlify(self.certificate_claim_id))
|
||||
|
||||
return self.validate_signature(self.HASHFUNC(to_sign).digest(), signature)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
FROM debian:buster-slim
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get upgrade -y && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
git \
|
||||
python3.7 \
|
||||
python3.7-dev \
|
||||
python3-pip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN python3.7 -m pip install --upgrade pip setuptools wheel
|
||||
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
|
||||
RUN python3.7 -m pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
RUN python3.7 -m pip install -e .
|
||||
|
||||
# Orchstr8 API
|
||||
EXPOSE 7954
|
||||
# Wallet Server
|
||||
EXPOSE 5280
|
||||
# SPV Server
|
||||
EXPOSE 50002
|
||||
# blockchain
|
||||
EXPOSE 9246
|
||||
ENV TORBA_LEDGER lbrynet.extras.wallet
|
||||
|
||||
RUN /usr/local/bin/orchstr8 download
|
||||
ENTRYPOINT ["/usr/local/bin/orchstr8"]
|
||||
@@ -0,0 +1,49 @@
|
||||
FROM debian:buster-slim
|
||||
|
||||
ARG user=lbry
|
||||
|
||||
# create an unprivileged user
|
||||
RUN groupadd -g 999 $user && useradd -r -u 999 -g $user $user
|
||||
RUN mkdir -p /home/$user/
|
||||
RUN chown -R $user:$user /home/$user/
|
||||
|
||||
# install python, pip, git and clean up
|
||||
RUN apt-get update && apt-get -y --no-install-recommends install build-essential git python3.7 python3.7-dev python3-pip && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# create and chown database dir
|
||||
ARG db_dir=/database
|
||||
RUN mkdir -p $db_dir
|
||||
RUN chown -R $user:$user $db_dir
|
||||
|
||||
# change into our user
|
||||
USER $user
|
||||
WORKDIR /home/$user
|
||||
|
||||
# RUN pip3 install -U pip (broken on pip 10 https://github.com/pypa/pip/issues/5240)
|
||||
RUN python3.7 -m pip install --upgrade pip setuptools
|
||||
|
||||
# get torba
|
||||
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git#egg=torba
|
||||
|
||||
# copy lbrynet
|
||||
RUN mkdir projects/
|
||||
COPY . projects/lbry
|
||||
USER root
|
||||
RUN chown -R $user:$user .
|
||||
USER $user
|
||||
WORKDIR projects/lbry
|
||||
RUN python3.7 -m pip install --user -e . && rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
ARG host=0.0.0.0
|
||||
ARG tcp_port=50001
|
||||
ARG daemon_url=http://lbry:lbry@localhost:9245/
|
||||
VOLUME $db_dir
|
||||
ENV TCP_PORT=$tcp_port
|
||||
ENV HOST=$host
|
||||
ENV DAEMON_URL=$daemon_url
|
||||
ENV DB_DIRECTORY=$db_dir
|
||||
ENV BANDWIDTH_LIMIT=1000000000000000000000000000000000000000000
|
||||
ENV MAX_SESSIONS=1000000000
|
||||
ENV MAX_SEND=1000000000000000000
|
||||
ENTRYPOINT ["/home/lbry/.local/bin/torba-server"]
|
||||
@@ -1,57 +0,0 @@
|
||||
"""Decrypt a single blob"""
|
||||
import argparse
|
||||
import binascii
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import reactor
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.cryptstream import CryptBlob
|
||||
from lbrynet.blob import BlobFile
|
||||
from lbrynet.core import log_support
|
||||
|
||||
|
||||
log = logging.getLogger('decrypt_blob')
|
||||
|
||||
|
||||
def main():
|
||||
conf.initialize_settings()
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('blob_file')
|
||||
parser.add_argument('hex_key')
|
||||
parser.add_argument('hex_iv')
|
||||
parser.add_argument('output')
|
||||
args = parser.parse_args()
|
||||
log_support.configure_console()
|
||||
|
||||
d = run(args)
|
||||
reactor.run()
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run(args):
|
||||
try:
|
||||
yield decrypt_blob(args.blob_file, args.hex_key, args.hex_iv, args.output)
|
||||
except Exception:
|
||||
log.exception('Failed to decrypt blob')
|
||||
finally:
|
||||
reactor.callLater(0, reactor.stop)
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def decrypt_blob(blob_file, key, iv, output):
|
||||
filename = os.path.abspath(blob_file)
|
||||
length = os.path.getsize(filename)
|
||||
directory, blob_hash = os.path.split(filename)
|
||||
blob = BlobFile(directory, blob_hash, length)
|
||||
decryptor = CryptBlob.StreamBlobDecryptor(
|
||||
blob, binascii.unhexlify(key), binascii.unhexlify(iv), length)
|
||||
with open(output, 'w') as f:
|
||||
yield decryptor.decrypt(f.write)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,85 +0,0 @@
|
||||
import curses
|
||||
import time
|
||||
import datetime
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
stdscr = curses.initscr()
|
||||
|
||||
api = JSONRPCProxy.from_url("http://localhost:5280")
|
||||
|
||||
|
||||
def init_curses():
|
||||
curses.noecho()
|
||||
curses.cbreak()
|
||||
stdscr.nodelay(1)
|
||||
stdscr.keypad(1)
|
||||
|
||||
|
||||
def teardown_curses():
|
||||
curses.nocbreak()
|
||||
stdscr.keypad(0)
|
||||
curses.echo()
|
||||
curses.endwin()
|
||||
|
||||
|
||||
def refresh(node_index):
|
||||
height, width = stdscr.getmaxyx()
|
||||
node_ids = api.get_node_ids()
|
||||
node_id = node_ids[node_index]
|
||||
node_statuses = api.node_status()
|
||||
running = node_statuses[node_id]
|
||||
buckets = api.node_routing_table(node_id=node_id)
|
||||
|
||||
for y in range(height):
|
||||
stdscr.addstr(y, 0, " " * (width - 1))
|
||||
|
||||
stdscr.addstr(0, 0, "node id: %s, running: %s (%i/%i running)" % (node_id, running, sum(node_statuses.values()), len(node_ids)))
|
||||
stdscr.addstr(1, 0, "%i buckets, %i contacts" %
|
||||
(len(buckets), sum([len(buckets[b]['contacts']) for b in buckets])))
|
||||
|
||||
y = 3
|
||||
for i in sorted(buckets.keys()):
|
||||
stdscr.addstr(y, 0, "bucket %s" % i)
|
||||
y += 1
|
||||
for h in sorted(buckets[i]['contacts'], key=lambda x: x['node_id'].decode('hex')):
|
||||
stdscr.addstr(y, 0, '%s (%s:%i) failures: %i, last replied to us: %s, last requested from us: %s' %
|
||||
(h['node_id'], h['address'], h['port'], h['failedRPCs'],
|
||||
datetime.datetime.fromtimestamp(float(h['lastReplied'] or 0)),
|
||||
datetime.datetime.fromtimestamp(float(h['lastRequested'] or 0))))
|
||||
y += 1
|
||||
y += 1
|
||||
|
||||
stdscr.addstr(y + 1, 0, str(time.time()))
|
||||
stdscr.refresh()
|
||||
return len(node_ids)
|
||||
|
||||
|
||||
def do_main():
|
||||
c = None
|
||||
nodes = 1
|
||||
node_index = 0
|
||||
while c not in [ord('q'), ord('Q')]:
|
||||
try:
|
||||
nodes = refresh(node_index)
|
||||
except:
|
||||
pass
|
||||
c = stdscr.getch()
|
||||
if c == curses.KEY_LEFT:
|
||||
node_index -= 1
|
||||
node_index = max(node_index, 0)
|
||||
elif c == curses.KEY_RIGHT:
|
||||
node_index += 1
|
||||
node_index = min(node_index, nodes - 1)
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
init_curses()
|
||||
do_main()
|
||||
finally:
|
||||
teardown_curses()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loaded 100 of 137 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user