Compare commits
95
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5971be5bef | ||
|
|
677d76a3e4 | ||
|
|
6ed3207cb7 | ||
|
|
be6448b74f | ||
|
|
b707ee4844 | ||
|
|
0e9888be3b | ||
|
|
68b56b7858 | ||
|
|
d32697f611 | ||
|
|
3c54b2bcd7 | ||
|
|
899b182811 | ||
|
|
0dfb5e008d | ||
|
|
abf94357eb | ||
|
|
36aded3830 | ||
|
|
d7ebf50602 | ||
|
|
523c4c86a9 | ||
|
|
74cccdbfc7 | ||
|
|
70471eebfa | ||
|
|
6339224980 | ||
|
|
fe97f7a119 | ||
|
|
f8ed605da2 | ||
|
|
e38ef2af1f | ||
|
|
5d8e627d02 | ||
|
|
c9d4ad5450 | ||
|
|
31586bf588 | ||
|
|
974d3c83e3 | ||
|
|
8dc8ca382b | ||
|
|
1dbb250413 | ||
|
|
9f9ec692c7 | ||
|
|
2c1b54c76c | ||
|
|
7314494fb9 | ||
|
|
e74a5e8ef3 | ||
|
|
b9af999739 | ||
|
|
2de155f90f | ||
|
|
ab3db2e626 | ||
|
|
f9d26b04d8 | ||
|
|
24e0673198 | ||
|
|
833261801f | ||
|
|
6c8cbc1b9e | ||
|
|
6ae4e68d1c | ||
|
|
d22a2dee33 | ||
|
|
0f1ed589a5 | ||
|
|
b032e99dd1 | ||
|
|
fd7811f38d | ||
|
|
071db3a73d | ||
|
|
950c122850 | ||
|
|
e0386ea9dc | ||
|
|
6c3eee0f0e | ||
|
|
4c23add193 | ||
|
|
248d1a18c7 | ||
|
|
17d1b85423 | ||
|
|
b0deb63129 | ||
|
|
ebdb33bd11 | ||
|
|
69ccbda168 | ||
|
|
a38643cca6 | ||
|
|
712b19605e | ||
|
|
8a5f3842fc | ||
|
|
db3e578bcd | ||
|
|
7e22adff7c | ||
|
|
76fbaac658 | ||
|
|
c4fac29ecb | ||
|
|
33392b3351 | ||
|
|
0613cf2d59 | ||
|
|
a930ca0631 | ||
|
|
63a7271008 | ||
|
|
5ca0ebcbb2 | ||
|
|
2c84b7a7dd | ||
|
|
71a0c704bd | ||
|
|
d30e7cd29e | ||
|
|
aeb7c6fe76 | ||
|
|
134c20da93 | ||
|
|
893b4fc3ca | ||
|
|
9b27e81940 | ||
|
|
72c0acf269 | ||
|
|
7be53ad692 | ||
|
|
16e596ec87 | ||
|
|
3fdcb80960 | ||
|
|
26a3221bac | ||
|
|
eb16cb0aa8 | ||
|
|
b5340505e5 | ||
|
|
4bb8129bb7 | ||
|
|
7f838d3a31 | ||
|
|
c186c31bf7 | ||
|
|
86fb99a37c | ||
|
|
6c52450858 | ||
|
|
fb4e94b04a | ||
|
|
248baf58b4 | ||
|
|
0845d65f4e | ||
|
|
dcc5415648 | ||
|
|
3c38a88b22 | ||
|
|
187b7670b1 | ||
|
|
049c17ff08 | ||
|
|
8de97bb441 | ||
|
|
97c0092923 | ||
|
|
e65e1fb5a2 | ||
|
|
20dd1e55f5 |
No files matched your search
@@ -0,0 +1,7 @@
|
||||
.git
|
||||
.tox
|
||||
__pycache__
|
||||
dist
|
||||
lbrynet.egg-info
|
||||
docs
|
||||
tests
|
||||
+6
-5
@@ -9,8 +9,8 @@ jobs:
|
||||
- stage: code quality
|
||||
name: "pylint lbrynet"
|
||||
install:
|
||||
- pip install astroid==2.0.4
|
||||
# newer astroid fails in pylint so we pre-install older version
|
||||
- pip install astroid==2.0.4 aiohttp==3.4.4
|
||||
# newer astroid and aiohttp fails in pylint so we pre-install older version
|
||||
- pip install pylint
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .
|
||||
@@ -24,8 +24,8 @@ jobs:
|
||||
- pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
- pip install -e .[test]
|
||||
script:
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m unittest discover -v tests.unit.wallet
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics tests.unit.components tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.test_cli tests.unit.test_customLogger tests.unit.schema
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.schema tests.unit.wallet tests.unit.test_customLogger tests.unit.components
|
||||
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.test_cli
|
||||
after_success:
|
||||
- coverage combine
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
@@ -110,7 +110,8 @@ jobs:
|
||||
env: OS=mac
|
||||
cache: false
|
||||
before_install:
|
||||
- brew upgrade python
|
||||
- brew upgrade python || true
|
||||
- brew upgrade python || true
|
||||
install:
|
||||
- python3 --version
|
||||
- pip3 --version
|
||||
|
||||
@@ -17,9 +17,6 @@ dht_node_port: 4444
|
||||
peer_port: 3333
|
||||
use_upnp: True
|
||||
|
||||
use_auth_http: True
|
||||
use_https: True
|
||||
|
||||
#components_to_skip:
|
||||
# - peer_protocol_server
|
||||
# - hash_announcer
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ from lbrynet.custom_logger import install_logger
|
||||
|
||||
|
||||
__name__ = "lbrynet"
|
||||
__version__ = "0.30.3"
|
||||
__version__ = "0.30.5"
|
||||
version = tuple(__version__.split('.'))
|
||||
|
||||
install_logger()
|
||||
|
||||
@@ -9,6 +9,7 @@ from binascii import hexlify
|
||||
from twisted.internet import defer
|
||||
from twisted.protocols.basic import FileSender
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorWriter, EncryptedFileStreamType
|
||||
from lbrynet.p2p.StreamDescriptor import format_sd_info, get_stream_hash, validate_descriptor
|
||||
from lbrynet.blob.CryptStreamCreator import CryptStreamCreator
|
||||
@@ -119,13 +120,13 @@ def create_lbry_file(blob_manager, storage, payment_rate_manager, lbry_file_mana
|
||||
sd_hash = yield descriptor_writer.create_descriptor(sd_info)
|
||||
|
||||
log.debug("saving the stream")
|
||||
yield storage.store_stream(
|
||||
yield f2d(storage.store_stream(
|
||||
sd_info['stream_hash'], sd_hash, sd_info['stream_name'], sd_info['key'],
|
||||
sd_info['suggested_file_name'], sd_info['blobs']
|
||||
)
|
||||
))
|
||||
log.debug("adding to the file manager")
|
||||
lbry_file = yield lbry_file_manager.add_published_file(
|
||||
lbry_file = yield f2d(lbry_file_manager.add_published_file(
|
||||
sd_info['stream_hash'], sd_hash, hexlify(file_directory.encode()), payment_rate_manager,
|
||||
payment_rate_manager.min_blob_data_payment_rate
|
||||
)
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
@@ -6,6 +6,7 @@ from binascii import hexlify, unhexlify
|
||||
|
||||
from twisted.internet import defer
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
|
||||
from lbrynet.utils import short_hash
|
||||
@@ -70,13 +71,11 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
self.channel_name = claim_info['channel_name']
|
||||
self.metadata = claim_info['value']['stream']['metadata']
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claim_info(self, include_supports=True):
|
||||
claim_info = yield self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
async def get_claim_info(self, include_supports=True):
|
||||
claim_info = await self.storage.get_content_claim(self.stream_hash, include_supports)
|
||||
if claim_info:
|
||||
self.set_claim_info(claim_info)
|
||||
|
||||
defer.returnValue(claim_info)
|
||||
return claim_info
|
||||
|
||||
@property
|
||||
def saving_status(self):
|
||||
@@ -106,11 +105,10 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
status = yield self._save_status()
|
||||
defer.returnValue(status)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def status(self):
|
||||
blobs = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
async def status(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
|
||||
completed_blobs = yield self.blob_manager.completed_blobs(blob_hashes)
|
||||
completed_blobs = self.blob_manager.completed_blobs(blob_hashes)
|
||||
num_blobs_completed = len(completed_blobs)
|
||||
num_blobs_known = len(blob_hashes)
|
||||
|
||||
@@ -120,9 +118,10 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
status = "stopped"
|
||||
else:
|
||||
status = "running"
|
||||
defer.returnValue(EncryptedFileStatusReport(
|
||||
|
||||
return EncryptedFileStatusReport(
|
||||
self.file_name, num_blobs_completed, num_blobs_known, status
|
||||
))
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start(self):
|
||||
@@ -150,7 +149,7 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_RUNNING
|
||||
status = yield self.lbry_file_manager.change_lbry_file_status(self, status)
|
||||
self._saving_status = False
|
||||
defer.returnValue(status)
|
||||
return status
|
||||
|
||||
def save_status(self):
|
||||
return self._save_status()
|
||||
@@ -180,10 +179,10 @@ class ManagedEncryptedFileDownloaderFactory:
|
||||
if file_name:
|
||||
file_name = hexlify(file_name.encode())
|
||||
hex_download_directory = hexlify(download_directory.encode())
|
||||
lbry_file = yield self.lbry_file_manager.add_downloaded_file(
|
||||
lbry_file = yield f2d(self.lbry_file_manager.add_downloaded_file(
|
||||
stream_hash, metadata.source_blob_hash, hex_download_directory, payment_rate_manager,
|
||||
data_rate, file_name=file_name, download_mirrors=download_mirrors
|
||||
)
|
||||
))
|
||||
defer.returnValue(lbry_file)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -8,6 +8,7 @@ from binascii import hexlify, unhexlify
|
||||
from twisted.internet import defer, task, reactor
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.reflector.reupload import reflect_file
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
@@ -41,11 +42,9 @@ class EncryptedFileManager:
|
||||
self.lbry_files = []
|
||||
self.lbry_file_reflector = task.LoopingCall(self.reflect_lbry_files)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
yield self._add_to_sd_identifier()
|
||||
yield self._start_lbry_files()
|
||||
log.info("Started file manager")
|
||||
self._add_to_sd_identifier()
|
||||
return self._start_lbry_files()
|
||||
|
||||
def get_lbry_file_status(self, lbry_file):
|
||||
return self.storage.get_lbry_file_status(lbry_file.rowid)
|
||||
@@ -55,7 +54,7 @@ class EncryptedFileManager:
|
||||
|
||||
def change_lbry_file_status(self, lbry_file, status):
|
||||
log.debug("Changing status of %s to %s", lbry_file.stream_hash, status)
|
||||
return self.storage.change_file_status(lbry_file.rowid, status)
|
||||
return f2d(self.storage.change_file_status(lbry_file.rowid, status))
|
||||
|
||||
def get_lbry_file_status_reports(self):
|
||||
ds = []
|
||||
@@ -115,10 +114,9 @@ class EncryptedFileManager:
|
||||
except Exception:
|
||||
log.warning("Failed to start %i", file_info.get('rowid'))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start_lbry_files(self):
|
||||
files = yield self.storage.get_all_lbry_files()
|
||||
claim_infos = yield self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
async def _start_lbry_files(self):
|
||||
files = await self.storage.get_all_lbry_files()
|
||||
claim_infos = await self.storage.get_claims_from_stream_hashes([file['stream_hash'] for file in files])
|
||||
prm = self.payment_rate_manager
|
||||
|
||||
log.info("Starting %i files", len(files))
|
||||
@@ -151,14 +149,13 @@ class EncryptedFileManager:
|
||||
log.info("Stopping %i lbry files", len(self.lbry_files))
|
||||
yield defer.DeferredList([self._stop_lbry_file(lbry_file) for lbry_file in list(self.lbry_files)])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
async def add_published_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager, blob_data_rate):
|
||||
status = ManagedEncryptedFileDownloader.STATUS_FINISHED
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = stream_metadata['suggested_file_name']
|
||||
rowid = yield self.storage.save_published_file(
|
||||
rowid = await self.storage.save_published_file(
|
||||
stream_hash, file_name, download_directory, blob_data_rate, status
|
||||
)
|
||||
lbry_file = self._get_lbry_file(
|
||||
@@ -166,37 +163,36 @@ class EncryptedFileManager:
|
||||
stream_metadata['suggested_file_name'], download_mirrors=None
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info()
|
||||
await lbry_file.get_claim_info()
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
async def add_downloaded_file(self, stream_hash, sd_hash, download_directory, payment_rate_manager=None,
|
||||
blob_data_rate=None, status=None, file_name=None, download_mirrors=None):
|
||||
status = status or ManagedEncryptedFileDownloader.STATUS_STOPPED
|
||||
payment_rate_manager = payment_rate_manager or self.payment_rate_manager
|
||||
blob_data_rate = blob_data_rate or payment_rate_manager.min_blob_data_payment_rate
|
||||
stream_metadata = yield get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
stream_metadata = await get_sd_info(self.storage, stream_hash, include_blobs=False)
|
||||
key = stream_metadata['key']
|
||||
stream_name = stream_metadata['stream_name']
|
||||
file_name = file_name or stream_metadata['suggested_file_name']
|
||||
|
||||
# when we save the file we'll atomic touch the nearest file to the suggested file name
|
||||
# that doesn't yet exist in the download directory
|
||||
rowid = yield self.storage.save_downloaded_file(
|
||||
rowid = await self.storage.save_downloaded_file(
|
||||
stream_hash, hexlify(os.path.basename(unhexlify(file_name))), download_directory, blob_data_rate
|
||||
)
|
||||
file_name = (yield self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
file_name = (await self.storage.get_filename_for_rowid(rowid)).decode()
|
||||
lbry_file = self._get_lbry_file(
|
||||
rowid, stream_hash, payment_rate_manager, sd_hash, key, stream_name, file_name, download_directory,
|
||||
stream_metadata['suggested_file_name'], download_mirrors
|
||||
)
|
||||
lbry_file.restore(status)
|
||||
yield lbry_file.get_claim_info(include_supports=False)
|
||||
await lbry_file.get_claim_info(include_supports=False)
|
||||
self.storage.content_claim_callbacks[stream_hash] = lbry_file.get_claim_info
|
||||
self.lbry_files.append(lbry_file)
|
||||
defer.returnValue(lbry_file)
|
||||
return lbry_file
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_lbry_file(self, lbry_file, delete_file=False):
|
||||
@@ -222,7 +218,7 @@ class EncryptedFileManager:
|
||||
del self.storage.content_claim_callbacks[lbry_file.stream_hash]
|
||||
|
||||
yield lbry_file.delete_data()
|
||||
yield self.storage.delete_stream(lbry_file.stream_hash)
|
||||
yield f2d(self.storage.delete_stream(lbry_file.stream_hash))
|
||||
|
||||
if delete_file and os.path.isfile(full_path):
|
||||
os.remove(full_path)
|
||||
@@ -240,7 +236,7 @@ class EncryptedFileManager:
|
||||
def reflect_lbry_files(self):
|
||||
sem = defer.DeferredSemaphore(self.CONCURRENT_REFLECTS)
|
||||
ds = []
|
||||
sd_hashes_to_reflect = yield self.storage.get_streams_to_re_reflect()
|
||||
sd_hashes_to_reflect = yield f2d(self.storage.get_streams_to_re_reflect())
|
||||
for lbry_file in self.lbry_files:
|
||||
if lbry_file.sd_hash in sd_hashes_to_reflect:
|
||||
ds.append(sem.run(reflect_file, lbry_file))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import logging
|
||||
from twisted.internet import defer, threads
|
||||
from twisted.internet import defer
|
||||
from twisted.web.client import FileBodyProducer
|
||||
from twisted.python.failure import Failure
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
@@ -111,23 +111,13 @@ class BlobFile:
|
||||
if not self.writers and not self.readers:
|
||||
self._verified = False
|
||||
self.saved_verified_blob = False
|
||||
|
||||
def delete_from_file_system():
|
||||
try:
|
||||
if os.path.isfile(self.file_path):
|
||||
os.remove(self.file_path)
|
||||
|
||||
d = threads.deferToThread(delete_from_file_system)
|
||||
|
||||
def log_error(err):
|
||||
log.warning("An error occurred deleting %s: %s",
|
||||
str(self.file_path), err.getErrorMessage())
|
||||
return err
|
||||
|
||||
d.addErrback(log_error)
|
||||
return d
|
||||
except Exception as e:
|
||||
log.exception("An error occurred deleting %s:", str(self.file_path), exc_info=e)
|
||||
else:
|
||||
return defer.fail(Failure(
|
||||
ValueError("File is currently being read or written and cannot be deleted")))
|
||||
raise ValueError("File is currently being read or written and cannot be deleted")
|
||||
|
||||
@property
|
||||
def verified(self):
|
||||
|
||||
@@ -4,6 +4,7 @@ import traceback
|
||||
from binascii import hexlify, unhexlify
|
||||
from twisted.internet import defer, threads
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.StreamDescriptor import save_sd_info
|
||||
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
|
||||
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
|
||||
@@ -28,16 +29,15 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_data(self):
|
||||
crypt_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
crypt_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
blob_hashes = [b.blob_hash for b in crypt_infos if b.blob_hash]
|
||||
sd_hash = yield self.storage.get_sd_blob_hash_for_stream(self.stream_hash)
|
||||
sd_hash = yield f2d(self.storage.get_sd_blob_hash_for_stream(self.stream_hash))
|
||||
blob_hashes.append(sd_hash)
|
||||
yield self.blob_manager.delete_blobs(blob_hashes)
|
||||
|
||||
def stop(self, err=None):
|
||||
d = self._close_output()
|
||||
d.addCallback(lambda _: CryptStreamDownloader.stop(self, err=err))
|
||||
return d
|
||||
self._close_output()
|
||||
return CryptStreamDownloader.stop(self, err=err)
|
||||
|
||||
def _get_progress_manager(self, download_manager):
|
||||
return FullStreamProgressManager(self._finished_downloading,
|
||||
@@ -65,14 +65,9 @@ class EncryptedFileDownloader(CryptStreamDownloader):
|
||||
def _close_output(self):
|
||||
pass
|
||||
|
||||
def get_total_bytes(self):
|
||||
d = self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
|
||||
def calculate_size(blobs):
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
d.addCallback(calculate_size)
|
||||
return d
|
||||
async def get_total_bytes(self):
|
||||
blobs = await self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
return sum([b.length for b in blobs])
|
||||
|
||||
def get_total_bytes_cached(self):
|
||||
if self._calculated_total_bytes is None or self._calculated_total_bytes == 0:
|
||||
@@ -173,15 +168,11 @@ class EncryptedFileSaver(EncryptedFileDownloader):
|
||||
|
||||
def _close_output(self):
|
||||
self.file_handle, file_handle = None, self.file_handle
|
||||
|
||||
def close_file():
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
return threads.deferToThread(close_file)
|
||||
if file_handle is not None:
|
||||
name = file_handle.name
|
||||
file_handle.close()
|
||||
if self.completed is False:
|
||||
os.remove(name)
|
||||
|
||||
def _get_write_func(self):
|
||||
def write_func(data):
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -17,9 +18,8 @@ class EncryptedFileMetadataHandler:
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_initial_blobs(self):
|
||||
blob_infos = yield self.storage.get_blobs_for_stream(self.stream_hash)
|
||||
formatted_infos = self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
defer.returnValue(formatted_infos)
|
||||
blob_infos = yield f2d(self.storage.get_blobs_for_stream(self.stream_hash))
|
||||
return self._format_initial_blobs_for_download_manager(blob_infos)
|
||||
|
||||
def final_blob_num(self):
|
||||
return self._final_blob_num
|
||||
|
||||
+13
-7
@@ -35,9 +35,10 @@ MB = 2 ** 20
|
||||
DEFAULT_CONCURRENT_ANNOUNCERS = 10
|
||||
|
||||
DEFAULT_DHT_NODES = [
|
||||
('lbrynet1.lbry.io', 4444),
|
||||
('lbrynet2.lbry.io', 4444),
|
||||
('lbrynet3.lbry.io', 4444)
|
||||
('lbrynet1.lbry.io', 4444), # US EAST
|
||||
('lbrynet2.lbry.io', 4444), # US WEST
|
||||
('lbrynet3.lbry.io', 4444), # EU
|
||||
('lbrynet4.lbry.io', 4444) # ASIA
|
||||
]
|
||||
|
||||
settings_decoders = {
|
||||
@@ -249,8 +250,6 @@ ADJUSTABLE_SETTINGS = {
|
||||
'sd_download_timeout': (int, 3),
|
||||
'share_usage_data': (bool, True), # whether to share usage stats and diagnostic info with LBRY
|
||||
'peer_search_timeout': (int, 60),
|
||||
'use_auth_http': (bool, False),
|
||||
'use_https': (bool, False),
|
||||
'use_upnp': (bool, True),
|
||||
'use_keyring': (bool, False),
|
||||
'wallet': (str, LBRYUM_WALLET),
|
||||
@@ -287,8 +286,11 @@ class Config:
|
||||
self.default_data_dir, self.default_wallet_dir, self.default_download_dir = get_linux_directories()
|
||||
else:
|
||||
assert None not in [data_dir, wallet_dir, download_dir]
|
||||
if data_dir:
|
||||
self.default_data_dir = data_dir
|
||||
if wallet_dir:
|
||||
self.default_wallet_dir = wallet_dir
|
||||
if download_dir:
|
||||
self.default_download_dir = download_dir
|
||||
|
||||
self._data = {
|
||||
@@ -590,6 +592,10 @@ class Config:
|
||||
if not os.path.isdir(self.wallet_dir):
|
||||
os.makedirs(self.wallet_dir)
|
||||
|
||||
def ensure_download_dir(self):
|
||||
if not os.path.isdir(self.download_dir):
|
||||
os.makedirs(self.download_dir)
|
||||
|
||||
def get_log_filename(self):
|
||||
"""
|
||||
Return the log file for this platform.
|
||||
@@ -598,8 +604,7 @@ class Config:
|
||||
return os.path.join(self.ensure_data_dir(), self['LOG_FILE_NAME'])
|
||||
|
||||
def get_api_connection_string(self, user: str = None, password: str = None) -> str:
|
||||
return 'http%s://%s%s:%i/%s' % (
|
||||
"" if not self['use_https'] else "s",
|
||||
return 'http://%s%s:%i/%s' % (
|
||||
"" if not (user and password) else f"{user}:{password}@",
|
||||
self['api_host'],
|
||||
self['api_port'],
|
||||
@@ -667,3 +672,4 @@ def initialize_settings(load_conf_file: typing.Optional[bool] = True,
|
||||
settings['wallet_dir'] = settings.wallet_dir or settings.default_wallet_dir
|
||||
settings.ensure_data_dir()
|
||||
settings.ensure_wallet_dir()
|
||||
settings.ensure_download_dir()
|
||||
@@ -32,8 +32,7 @@ from textwrap import dedent
|
||||
from lbrynet import conf, log_support, __name__ as lbrynet_name
|
||||
from lbrynet.utils import check_connection, json_dumps_pretty
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
from lbrynet.extras.daemon.DaemonConsole import main as daemon_console
|
||||
from lbrynet.extras.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.extras.daemon.DaemonConsole import main as daemon_console, LBRYAPIClient
|
||||
from lbrynet.extras.system_info import get_platform
|
||||
|
||||
log = logging.getLogger(lbrynet_name)
|
||||
@@ -58,7 +57,7 @@ def start_daemon(settings: typing.Optional[typing.Dict] = None,
|
||||
|
||||
if check_connection():
|
||||
daemon = Daemon()
|
||||
daemon.start_listening()
|
||||
reactor._asyncioEventloop.create_task(daemon.start_listening())
|
||||
reactor.run()
|
||||
else:
|
||||
log.info("Not connected to internet, unable to start")
|
||||
@@ -87,7 +86,8 @@ def start_daemon_with_cli_args(argv=None, data_dir: typing.Optional[str] = None,
|
||||
args = parser.parse_args(argv)
|
||||
settings = {}
|
||||
if args.useauth:
|
||||
settings['use_auth_http'] = True
|
||||
print('--http-auth is no longer supported; an alternative solution using IPC is forthcoming.')
|
||||
return
|
||||
|
||||
verbose = None
|
||||
if args.verbose:
|
||||
@@ -207,7 +207,7 @@ def main(argv=None):
|
||||
return 1
|
||||
|
||||
dir_args = {}
|
||||
if len(argv) > 2:
|
||||
if len(argv) >= 2:
|
||||
dir_arg_keys = [
|
||||
'data_dir',
|
||||
'wallet_dir',
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import asyncio
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
def d2f(deferred):
|
||||
return deferred.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def f2d(future):
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
@@ -36,37 +36,35 @@ class Component(metaclass=ComponentType):
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup(self):
|
||||
async def _setup(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.start)
|
||||
result = await self.start()
|
||||
self._running = True
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||
log.exception("Error setting up %s", self.component_name or self.__class__.__name__)
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop(self):
|
||||
async def _stop(self):
|
||||
try:
|
||||
result = yield defer.maybeDeferred(self.stop)
|
||||
result = await self.stop()
|
||||
self._running = False
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
except (defer.CancelledError, AlreadyQuit):
|
||||
pass
|
||||
except Exception as err:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
from lbrynet.p2p.Error import ComponentStartConditionNotMet
|
||||
from lbrynet.extras.daemon.PeerManager import PeerManager
|
||||
from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
|
||||
@@ -110,13 +110,8 @@ class ComponentManager:
|
||||
steps.reverse()
|
||||
return steps
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self, **callbacks):
|
||||
"""
|
||||
Start Components in sequence sorted by requirements
|
||||
|
||||
:return: (defer.Deferred)
|
||||
"""
|
||||
async def setup(self, **callbacks):
|
||||
""" Start Components in sequence sorted by requirements """
|
||||
for component_name, cb in callbacks.items():
|
||||
if component_name not in self.component_classes:
|
||||
if component_name not in self.skip_components:
|
||||
@@ -124,19 +119,22 @@ class ComponentManager:
|
||||
if not callable(cb):
|
||||
raise ValueError("%s is not callable" % cb)
|
||||
|
||||
def _setup(component):
|
||||
async def _setup(component):
|
||||
await component._setup()
|
||||
if component.component_name in callbacks:
|
||||
d = component._setup()
|
||||
d.addCallback(callbacks[component.component_name], component)
|
||||
return d
|
||||
return component._setup()
|
||||
maybe_coro = callbacks[component.component_name](component)
|
||||
if asyncio.iscoroutine(maybe_coro):
|
||||
asyncio.create_task(maybe_coro)
|
||||
|
||||
stages = self.sort_components()
|
||||
for stage in stages:
|
||||
yield defer.DeferredList([_setup(component) for component in stage if not component.running])
|
||||
needing_start = [
|
||||
_setup(component) for component in stage if not component.running
|
||||
]
|
||||
if needing_start:
|
||||
await asyncio.wait(needing_start)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
"""
|
||||
Stop Components in reversed startup order
|
||||
|
||||
@@ -144,7 +142,11 @@ class ComponentManager:
|
||||
"""
|
||||
stages = self.sort_components(reverse=True)
|
||||
for stage in stages:
|
||||
yield defer.DeferredList([component._stop() for component in stage if component.running])
|
||||
needing_stop = [
|
||||
component._stop() for component in stage if component.running
|
||||
]
|
||||
if needing_stop:
|
||||
await asyncio.wait(needing_stop)
|
||||
|
||||
def all_components_running(self, *component_names):
|
||||
"""
|
||||
|
||||
+102
-141
@@ -1,13 +1,13 @@
|
||||
import os
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import logging
|
||||
import treq
|
||||
import json
|
||||
import math
|
||||
import binascii
|
||||
from hashlib import sha256
|
||||
from types import SimpleNamespace
|
||||
from twisted.internet import defer, threads, reactor, error, task
|
||||
from twisted.internet import defer, reactor, error
|
||||
|
||||
from aioupnp import __version__ as aioupnp_version
|
||||
from aioupnp.upnp import UPnP
|
||||
@@ -16,6 +16,7 @@ from aioupnp.fault import UPnPError
|
||||
import lbrynet.schema
|
||||
from lbrynet import conf
|
||||
|
||||
from lbrynet.extras.compat import d2f
|
||||
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
|
||||
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
|
||||
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
|
||||
@@ -27,7 +28,7 @@ from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
|
||||
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.wallet import Network
|
||||
from lbrynet.utils import DeferredDict, generate_id
|
||||
from lbrynet.utils import generate_id
|
||||
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
|
||||
from lbrynet.p2p.RateLimiter import RateLimiter
|
||||
from lbrynet.p2p.BlobManager import DiskBlobManager
|
||||
@@ -55,22 +56,23 @@ RATE_LIMITER_COMPONENT = "rate_limiter"
|
||||
PAYMENT_RATE_COMPONENT = "payment_rate_manager"
|
||||
|
||||
|
||||
def from_future(coroutine: asyncio.coroutine) -> defer.Deferred:
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(coroutine))
|
||||
async def gather_dict(tasks: dict):
|
||||
async def wait_value(key, value):
|
||||
return key, await value
|
||||
return dict(await asyncio.gather(*(
|
||||
wait_value(*kv) for kv in tasks.items()
|
||||
)))
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
async def get_external_ip(): # used if upnp is disabled or non-functioning
|
||||
try:
|
||||
buf = []
|
||||
response = yield treq.get("https://api.lbry.io/ip")
|
||||
yield treq.collect(response, buf.append)
|
||||
parsed = json.loads(b"".join(buf).decode())
|
||||
if parsed['success']:
|
||||
return parsed['data']['ip']
|
||||
return
|
||||
except Exception as err:
|
||||
return
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get("https://api.lbry.io/ip") as resp:
|
||||
response = await resp.json()
|
||||
if response['success']:
|
||||
return response['data']['ip']
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseComponent(Component):
|
||||
@@ -97,8 +99,7 @@ class DatabaseComponent(Component):
|
||||
with open(conf.settings.get_db_revision_filename(), mode='w') as db_revision:
|
||||
db_revision.write(str(version_num))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
# check directories exist, create them if they don't
|
||||
log.info("Loading databases")
|
||||
|
||||
@@ -117,19 +118,19 @@ class DatabaseComponent(Component):
|
||||
if old_revision < self.get_current_db_revision():
|
||||
from lbrynet.extras.daemon.migrator import dbmigrator
|
||||
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
|
||||
yield threads.deferToThread(
|
||||
dbmigrator.migrate_db, conf.settings.data_dir, old_revision, self.get_current_db_revision()
|
||||
await asyncio.get_event_loop().run_in_executor(
|
||||
None, dbmigrator.migrate_db, conf.settings.data_dir, old_revision, self.get_current_db_revision()
|
||||
)
|
||||
self._write_db_revision_file(self.get_current_db_revision())
|
||||
log.info("Finished upgrading the databases.")
|
||||
|
||||
# start SQLiteStorage
|
||||
self.storage = SQLiteStorage(conf.settings.data_dir)
|
||||
yield self.storage.setup()
|
||||
self.storage = SQLiteStorage(
|
||||
os.path.join(conf.settings.data_dir, "lbrynet.sqlite")
|
||||
)
|
||||
await self.storage.open()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.storage.stop()
|
||||
async def stop(self):
|
||||
await self.storage.close()
|
||||
self.storage = None
|
||||
|
||||
|
||||
@@ -152,7 +153,7 @@ class HeadersComponent(Component):
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {} if not self._downloading_headers else {
|
||||
'downloading_headers': self._downloading_headers,
|
||||
'download_progress': self._headers_progress_percent
|
||||
@@ -250,33 +251,24 @@ class HeadersComponent(Component):
|
||||
with open(self.headers_file, "rb+") as headers_file:
|
||||
headers_file.truncate(checksum_length_in_bytes)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
conf.settings.ensure_wallet_dir()
|
||||
if not os.path.exists(self.headers_dir):
|
||||
os.mkdir(self.headers_dir)
|
||||
if os.path.exists(self.old_file):
|
||||
log.warning("Moving old headers from %s to %s.", self.old_file, self.headers_file)
|
||||
os.rename(self.old_file, self.headers_file)
|
||||
self._downloading_headers = yield f2d(self.should_download_headers_from_s3())
|
||||
self._downloading_headers = await self.should_download_headers_from_s3()
|
||||
if self._downloading_headers:
|
||||
try:
|
||||
yield self.fetch_headers_from_s3()
|
||||
await d2f(self.fetch_headers_from_s3())
|
||||
except Exception as err:
|
||||
log.error("failed to fetch headers from s3: %s", err)
|
||||
finally:
|
||||
self._downloading_headers = False
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
def d2f(deferred):
|
||||
return deferred.asFuture(asyncio.get_event_loop())
|
||||
|
||||
|
||||
def f2d(future):
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class WalletComponent(Component):
|
||||
@@ -291,7 +283,7 @@ class WalletComponent(Component):
|
||||
def component(self):
|
||||
return self.wallet_manager
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
if self.wallet_manager and self.running:
|
||||
local_height = self.wallet_manager.network.get_local_height()
|
||||
remote_height = self.wallet_manager.network.get_server_height()
|
||||
@@ -304,19 +296,17 @@ class WalletComponent(Component):
|
||||
'is_locked': not self.wallet_manager.is_wallet_unlocked,
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
conf.settings.ensure_wallet_dir()
|
||||
log.info("Starting torba wallet")
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = conf.settings['blockchain_name']
|
||||
self.wallet_manager = yield f2d(LbryWalletManager.from_lbrynet_config(conf.settings, storage))
|
||||
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(conf.settings, storage)
|
||||
self.wallet_manager.old_db = storage
|
||||
yield f2d(self.wallet_manager.start())
|
||||
await self.wallet_manager.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield f2d(self.wallet_manager.stop())
|
||||
async def stop(self):
|
||||
await self.wallet_manager.stop()
|
||||
self.wallet_manager = None
|
||||
|
||||
|
||||
@@ -345,14 +335,11 @@ class BlobComponent(Component):
|
||||
def stop(self):
|
||||
return self.blob_manager.stop()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
count = 0
|
||||
if self.blob_manager:
|
||||
count = yield self.blob_manager.storage.count_finished_blobs()
|
||||
defer.returnValue({
|
||||
'finished_blobs': count
|
||||
})
|
||||
count = await self.blob_manager.storage.count_finished_blobs()
|
||||
return {'finished_blobs': count}
|
||||
|
||||
|
||||
class DHTComponent(Component):
|
||||
@@ -370,14 +357,13 @@ class DHTComponent(Component):
|
||||
def component(self):
|
||||
return self.dht_node
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {
|
||||
'node_id': binascii.hexlify(conf.settings.get_node_id()),
|
||||
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", conf.settings["peer_port"])
|
||||
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", conf.settings["dht_node_port"])
|
||||
@@ -387,7 +373,7 @@ class DHTComponent(Component):
|
||||
external_ip = self.upnp_component.external_ip
|
||||
if not external_ip:
|
||||
log.warning("UPnP component failed to get external ip")
|
||||
external_ip = yield get_external_ip()
|
||||
external_ip = await get_external_ip()
|
||||
if not external_ip:
|
||||
log.warning("failed to get external ip")
|
||||
|
||||
@@ -399,12 +385,11 @@ class DHTComponent(Component):
|
||||
peerPort=self.external_peer_port
|
||||
)
|
||||
|
||||
yield self.dht_node.start(conf.settings['known_dht_nodes'], block_on_join=False)
|
||||
await d2f(self.dht_node.start(conf.settings['known_dht_nodes'], block_on_join=False))
|
||||
log.info("Started the dht")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.dht_node.stop()
|
||||
return d2f(self.dht_node.stop())
|
||||
|
||||
|
||||
class HashAnnouncerComponent(Component):
|
||||
@@ -419,18 +404,16 @@ class HashAnnouncerComponent(Component):
|
||||
def component(self):
|
||||
return self.hash_announcer
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
storage = self.component_manager.get_component(DATABASE_COMPONENT)
|
||||
dht_node = self.component_manager.get_component(DHT_COMPONENT)
|
||||
self.hash_announcer = DHTHashAnnouncer(dht_node, storage)
|
||||
yield self.hash_announcer.start()
|
||||
self.hash_announcer.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.hash_announcer.stop()
|
||||
self.hash_announcer.stop()
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {
|
||||
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.hash_queue)
|
||||
}
|
||||
@@ -447,13 +430,11 @@ class RateLimiterComponent(Component):
|
||||
def component(self):
|
||||
return self.rate_limiter
|
||||
|
||||
def start(self):
|
||||
async def start(self):
|
||||
self.rate_limiter.start()
|
||||
return defer.succeed(None)
|
||||
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
self.rate_limiter.stop()
|
||||
return defer.succeed(None)
|
||||
|
||||
|
||||
class PaymentRateComponent(Component):
|
||||
@@ -467,11 +448,11 @@ class PaymentRateComponent(Component):
|
||||
def component(self):
|
||||
return self.payment_rate_manager
|
||||
|
||||
def start(self):
|
||||
return defer.succeed(None)
|
||||
async def start(self):
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class FileManagerComponent(Component):
|
||||
@@ -487,14 +468,13 @@ class FileManagerComponent(Component):
|
||||
def component(self):
|
||||
return self.file_manager
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
if not self.file_manager:
|
||||
return
|
||||
return {
|
||||
'managed_files': len(self.file_manager.lbry_files)
|
||||
}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
@@ -511,18 +491,16 @@ class FileManagerComponent(Component):
|
||||
wallet,
|
||||
conf.settings.download_dir
|
||||
)
|
||||
yield sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
|
||||
sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
|
||||
|
||||
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
|
||||
log.info('Starting the file manager')
|
||||
self.file_manager = EncryptedFileManager(self.component_manager.peer_finder, rate_limiter, blob_manager, wallet,
|
||||
payment_rate_manager, storage, sd_identifier)
|
||||
yield self.file_manager.setup()
|
||||
log.info('Done setting up file manager')
|
||||
return self.file_manager.setup()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.file_manager.stop()
|
||||
return d2f(self.file_manager.stop())
|
||||
|
||||
|
||||
class PeerProtocolServerComponent(Component):
|
||||
@@ -538,8 +516,7 @@ class PeerProtocolServerComponent(Component):
|
||||
def component(self):
|
||||
return self.lbry_server_port
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
wallet = self.component_manager.get_component(WALLET_COMPONENT)
|
||||
upnp = self.component_manager.get_component(UPNP_COMPONENT)
|
||||
peer_port = conf.settings['peer_port']
|
||||
@@ -562,7 +539,7 @@ class PeerProtocolServerComponent(Component):
|
||||
try:
|
||||
log.info("Peer protocol listening on TCP %i (ext port %i)", peer_port,
|
||||
upnp.upnp_redirects.get("TCP", peer_port))
|
||||
self.lbry_server_port = yield reactor.listenTCP(peer_port, server_factory)
|
||||
self.lbry_server_port = reactor.listenTCP(peer_port, server_factory)
|
||||
except error.CannotListenError as e:
|
||||
import traceback
|
||||
log.error("Couldn't bind to port %d. Visit lbry.io/faq/how-to-change-port for"
|
||||
@@ -570,12 +547,11 @@ class PeerProtocolServerComponent(Component):
|
||||
log.error("%s", traceback.format_exc())
|
||||
raise ValueError("%s lbrynet may already be running on your computer." % str(e))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
if self.lbry_server_port is not None:
|
||||
self.lbry_server_port, old_port = None, self.lbry_server_port
|
||||
log.info('Stop listening on port %s', old_port.port)
|
||||
yield old_port.stopListening()
|
||||
await d2f(old_port.stopListening())
|
||||
|
||||
|
||||
class ReflectorComponent(Component):
|
||||
@@ -591,25 +567,23 @@ class ReflectorComponent(Component):
|
||||
def component(self):
|
||||
return self.reflector_server
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
log.info("Starting reflector server")
|
||||
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
|
||||
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
|
||||
reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager)
|
||||
try:
|
||||
self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory)
|
||||
self.reflector_server = await d2f(reactor.listenTCP(self.reflector_server_port, reflector_factory))
|
||||
log.info('Started reflector on port %s', self.reflector_server_port)
|
||||
except error.CannotListenError as e:
|
||||
log.exception("Couldn't bind reflector to port %d", self.reflector_server_port)
|
||||
raise ValueError(f"{e} lbrynet may already be running on your computer.")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
async def stop(self):
|
||||
if self.reflector_server is not None:
|
||||
log.info("Stopping reflector server")
|
||||
self.reflector_server, p = None, self.reflector_server
|
||||
yield p.stopListening
|
||||
await d2f(p.stopListening())
|
||||
|
||||
|
||||
class UPnPComponent(Component):
|
||||
@@ -623,29 +597,23 @@ class UPnPComponent(Component):
|
||||
self.upnp = None
|
||||
self.upnp_redirects = {}
|
||||
self.external_ip = None
|
||||
self._maintain_redirects_lc = task.LoopingCall(self._maintain_redirects)
|
||||
self._maintain_redirects_lc.clock = self.component_manager.reactor
|
||||
self._maintain_redirects_task = None
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup_redirects(self):
|
||||
d = {}
|
||||
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
|
||||
upnp_redirects = yield DeferredDict(d)
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
async def _repeatedly_maintain_redirects(self, now=True):
|
||||
while True:
|
||||
if now:
|
||||
await self._maintain_redirects()
|
||||
await asyncio.sleep(360)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _maintain_redirects(self):
|
||||
async def _maintain_redirects(self):
|
||||
# setup the gateway if necessary
|
||||
if not self.upnp:
|
||||
try:
|
||||
self.upnp = yield from_future(UPnP.discover())
|
||||
self.upnp = await UPnP.discover()
|
||||
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
except Exception as err:
|
||||
log.warning("upnp discovery failed: %s", err)
|
||||
@@ -655,7 +623,7 @@ class UPnPComponent(Component):
|
||||
external_ip = None
|
||||
if self.upnp:
|
||||
try:
|
||||
external_ip = yield from_future(self.upnp.get_external_ip())
|
||||
external_ip = await self.upnp.get_external_ip()
|
||||
if external_ip != "0.0.0.0" and not self.external_ip:
|
||||
log.info("got external ip from UPnP: %s", external_ip)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
@@ -663,7 +631,7 @@ class UPnPComponent(Component):
|
||||
|
||||
if external_ip == "0.0.0.0" or not external_ip:
|
||||
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
|
||||
external_ip = yield get_external_ip()
|
||||
external_ip = await get_external_ip()
|
||||
if self.external_ip and self.external_ip != external_ip:
|
||||
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
|
||||
self.external_ip = external_ip
|
||||
@@ -674,10 +642,10 @@ class UPnPComponent(Component):
|
||||
log.info("add UPnP port mappings")
|
||||
d = {}
|
||||
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
|
||||
d["TCP"] = self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
if DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
|
||||
upnp_redirects = yield DeferredDict(d)
|
||||
d["UDP"] = self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
upnp_redirects = await gather_dict(d)
|
||||
log.info("set up redirects: %s", upnp_redirects)
|
||||
self.upnp_redirects.update(upnp_redirects)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
@@ -685,7 +653,7 @@ class UPnPComponent(Component):
|
||||
return self._maintain_redirects()
|
||||
elif self.upnp: # check existing redirects are still active
|
||||
found = set()
|
||||
mappings = yield from_future(self.upnp.get_redirects())
|
||||
mappings = await self.upnp.get_redirects()
|
||||
for mapping in mappings:
|
||||
proto = mapping['NewProtocol']
|
||||
if proto in self.upnp_redirects and mapping['NewExternalPort'] == self.upnp_redirects[proto]:
|
||||
@@ -693,18 +661,14 @@ class UPnPComponent(Component):
|
||||
found.add(proto)
|
||||
if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
udp_port = yield from_future(
|
||||
self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
)
|
||||
udp_port = await self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
|
||||
self.upnp_redirects['UDP'] = udp_port
|
||||
log.info("refreshed upnp redirect for dht port: %i", udp_port)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
del self.upnp_redirects['UDP']
|
||||
if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
|
||||
try:
|
||||
tcp_port = yield from_future(
|
||||
self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
)
|
||||
tcp_port = await self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
|
||||
self.upnp_redirects['TCP'] = tcp_port
|
||||
log.info("refreshed upnp redirect for peer port: %i", tcp_port)
|
||||
except (asyncio.TimeoutError, UPnPError):
|
||||
@@ -715,14 +679,13 @@ class UPnPComponent(Component):
|
||||
if self.upnp_redirects:
|
||||
log.debug("upnp redirects are still active")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
async def start(self):
|
||||
log.info("detecting external ip")
|
||||
if not self.use_upnp:
|
||||
self.external_ip = yield get_external_ip()
|
||||
self.external_ip = await get_external_ip()
|
||||
return
|
||||
success = False
|
||||
yield self._maintain_redirects()
|
||||
await self._maintain_redirects()
|
||||
if self.upnp:
|
||||
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
|
||||
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
|
||||
@@ -733,18 +696,18 @@ class UPnPComponent(Component):
|
||||
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
|
||||
else:
|
||||
log.error("failed to setup upnp")
|
||||
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, self.get_status())
|
||||
self._maintain_redirects_lc.start(360, now=False)
|
||||
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
|
||||
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
|
||||
|
||||
def stop(self):
|
||||
if self._maintain_redirects_lc.running:
|
||||
self._maintain_redirects_lc.stop()
|
||||
return defer.DeferredList(
|
||||
[from_future(self.upnp.delete_port_mapping(port, protocol))
|
||||
for protocol, port in self.upnp_redirects.items()]
|
||||
)
|
||||
async def stop(self):
|
||||
if self.upnp_redirects:
|
||||
await asyncio.wait([
|
||||
self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
|
||||
])
|
||||
if self._maintain_redirects_task is not None and not self._maintain_redirects_task.done():
|
||||
self._maintain_redirects_task.cancel()
|
||||
|
||||
def get_status(self):
|
||||
async def get_status(self):
|
||||
return {
|
||||
'aioupnp_version': aioupnp_version,
|
||||
'redirects': self.upnp_redirects,
|
||||
@@ -766,10 +729,8 @@ class ExchangeRateManagerComponent(Component):
|
||||
def component(self):
|
||||
return self.exchange_rate_manager
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
yield self.exchange_rate_manager.start()
|
||||
async def start(self):
|
||||
self.exchange_rate_manager.start()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
yield self.exchange_rate_manager.stop()
|
||||
async def stop(self):
|
||||
self.exchange_rate_manager.stop()
|
||||
+531
-333
File diff suppressed because it is too large.
Load diff
@@ -8,10 +8,60 @@ from aiohttp import client_exceptions
|
||||
|
||||
from lbrynet import utils, conf, log_support
|
||||
from lbrynet.extras.daemon import analytics
|
||||
from lbrynet.extras.daemon.auth.client import LBRYAPIClient
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
import aiohttp
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
USER_AGENT = "AuthServiceProxy/0.1"
|
||||
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
|
||||
TWISTED_SESSION = "TWISTED_SESSION"
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
HTTP_TIMEOUT = 30
|
||||
|
||||
|
||||
class JSONRPCException(Exception):
|
||||
def __init__(self, rpc_error):
|
||||
super().__init__()
|
||||
self.error = rpc_error
|
||||
|
||||
|
||||
class UnAuthAPIClient:
|
||||
def __init__(self, host, port, session):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.session = session
|
||||
|
||||
def __getattr__(self, method):
|
||||
async def f(*args, **kwargs):
|
||||
return await self.call(method, [args, kwargs])
|
||||
|
||||
return f
|
||||
|
||||
@classmethod
|
||||
async def from_url(cls, url):
|
||||
url_fragment = urlparse(url)
|
||||
host = url_fragment.hostname
|
||||
port = url_fragment.port
|
||||
connector = aiohttp.TCPConnector()
|
||||
session = aiohttp.ClientSession(connector=connector)
|
||||
return cls(host, port, session)
|
||||
|
||||
async def call(self, method, params=None):
|
||||
message = {'method': method, 'params': params}
|
||||
async with self.session.get(conf.settings.get_api_connection_string(), json=message) as resp:
|
||||
return await resp.json()
|
||||
|
||||
|
||||
class LBRYAPIClient:
|
||||
@staticmethod
|
||||
def get_client(conf_path=None):
|
||||
conf.conf_file = conf_path
|
||||
if not conf.settings:
|
||||
conf.initialize_settings()
|
||||
return UnAuthAPIClient.from_url(conf.settings.get_api_connection_string())
|
||||
|
||||
|
||||
if sys.platform.startswith('darwin') or sys.platform.startswith('linux'):
|
||||
@@ -169,7 +219,8 @@ def start_server_and_listen(use_auth, analytics_manager, quiet):
|
||||
logging.getLogger("lbryum").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("requests").setLevel(logging.CRITICAL)
|
||||
|
||||
analytics_manager.send_server_startup()
|
||||
# TODO: turn this all into async. Until then this routine can't be called
|
||||
# analytics_manager.send_server_startup()
|
||||
yield Daemon().start_listening()
|
||||
|
||||
|
||||
@@ -221,6 +272,9 @@ def main():
|
||||
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.useauth:
|
||||
print('--http-auth is no longer supported; an alternative solution using IPC is forthcoming.')
|
||||
return
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(start_lbrynet_console(args.quiet, args.use_existing_daemon, args.useauth))
|
||||
reactor.run()
|
||||
|
||||
@@ -11,7 +11,7 @@ from lbrynet.p2p.StreamDescriptor import download_sd_blob
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
|
||||
from torba.client.constants import COIN
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc
|
||||
from lbrynet.extras.daemon.Components import f2d
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
INITIALIZING_CODE = 'initializing'
|
||||
DOWNLOAD_METADATA_CODE = 'downloading_metadata'
|
||||
@@ -162,7 +162,7 @@ class GetStream:
|
||||
def _download(self, sd_blob, name, key_fee, txid, nout, file_name=None):
|
||||
self.downloader = yield self._create_downloader(sd_blob, file_name=file_name)
|
||||
yield self.pay_key_fee(key_fee, name)
|
||||
yield self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout))
|
||||
yield f2d(self.storage.save_content_claim(self.downloader.stream_hash, "%s:%i" % (txid, nout)))
|
||||
self.finished_deferred = self.downloader.start()
|
||||
self.downloader.download_manager.progress_manager.wrote_first_data.addCallback(
|
||||
self.data_downloading_deferred.callback
|
||||
|
||||
@@ -2,6 +2,7 @@ import binascii
|
||||
import logging
|
||||
|
||||
from twisted.internet import defer, task
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet import utils, conf
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -38,7 +39,7 @@ class DHTHashAnnouncer:
|
||||
now = self.clock.seconds()
|
||||
if storing_node_ids:
|
||||
result = (now, storing_node_ids)
|
||||
yield self.storage.update_last_announced_blob(blob_hash, now)
|
||||
yield f2d(self.storage.update_last_announced_blob(blob_hash, now))
|
||||
log.debug("Stored %s to %i peers", blob_hash[:16], len(storing_node_ids))
|
||||
else:
|
||||
result = (None, [])
|
||||
@@ -82,7 +83,7 @@ class DHTHashAnnouncer:
|
||||
if not self.dht_node.contacts:
|
||||
log.info("Not ready to start announcing hashes")
|
||||
return
|
||||
need_reannouncement = yield self.storage.get_blobs_to_announce()
|
||||
need_reannouncement = yield f2d(self.storage.get_blobs_to_announce())
|
||||
if need_reannouncement:
|
||||
yield self.immediate_announce(need_reannouncement)
|
||||
else:
|
||||
|
||||
@@ -3,7 +3,7 @@ import logging
|
||||
import os
|
||||
|
||||
from lbrynet.blob.EncryptedFileCreator import create_lbry_file
|
||||
from lbrynet.extras.daemon.mime_types import guess_mime_type
|
||||
from lbrynet.extras.daemon.mime_types import guess_media_type
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -43,25 +43,25 @@ class Publisher:
|
||||
claim_dict['stream']['source'] = {}
|
||||
claim_dict['stream']['source']['source'] = self.lbry_file.sd_hash
|
||||
claim_dict['stream']['source']['sourceType'] = 'lbry_sd_hash'
|
||||
claim_dict['stream']['source']['contentType'] = guess_mime_type(file_path)
|
||||
claim_dict['stream']['source']['contentType'] = guess_media_type(file_path)
|
||||
claim_dict['stream']['source']['version'] = "_0_0_1" # need current version here
|
||||
tx = await self.wallet.claim_name(
|
||||
self.account, name, bid, claim_dict, self.certificate, holding_address
|
||||
)
|
||||
|
||||
# check if we have a file already for this claim (if this is a publish update with a new stream)
|
||||
old_stream_hashes = await d2f(self.storage.get_old_stream_hashes_for_claim_id(
|
||||
old_stream_hashes = await self.storage.get_old_stream_hashes_for_claim_id(
|
||||
tx.outputs[0].claim_id, self.lbry_file.stream_hash
|
||||
))
|
||||
)
|
||||
if old_stream_hashes:
|
||||
for lbry_file in filter(lambda l: l.stream_hash in old_stream_hashes,
|
||||
list(self.lbry_file_manager.lbry_files)):
|
||||
await d2f(self.lbry_file_manager.delete_lbry_file(lbry_file, delete_file=False))
|
||||
log.info("Removed old stream for claim update: %s", lbry_file.stream_hash)
|
||||
|
||||
await d2f(self.storage.save_content_claim(
|
||||
await self.storage.save_content_claim(
|
||||
self.lbry_file.stream_hash, tx.outputs[0].id
|
||||
))
|
||||
)
|
||||
return tx
|
||||
|
||||
async def publish_stream(self, name, bid, claim_dict, stream_hash, holding_address=None):
|
||||
@@ -70,8 +70,8 @@ class Publisher:
|
||||
self.account, name, bid, claim_dict, self.certificate, holding_address
|
||||
)
|
||||
if stream_hash: # the stream_hash returned from the db will be None if this isn't a stream we have
|
||||
await d2f(self.storage.save_content_claim(
|
||||
await self.storage.save_content_claim(
|
||||
stream_hash, tx.outputs[0].id
|
||||
))
|
||||
)
|
||||
self.lbry_file = [f for f in self.lbry_file_manager.lbry_files if f.stream_hash == stream_hash][0]
|
||||
return tx
|
||||
@@ -1,11 +1,12 @@
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import treq
|
||||
from twisted.internet import defer, task
|
||||
import asyncio
|
||||
import aiohttp
|
||||
|
||||
from lbrynet import conf, utils
|
||||
from lbrynet.extras import looping_call_manager, system_info
|
||||
from lbrynet.extras import system_info
|
||||
from lbrynet.extras.daemon.storage import looping_call
|
||||
|
||||
# Things We Track
|
||||
SERVER_STARTUP = 'Server Startup'
|
||||
@@ -30,7 +31,7 @@ class Manager:
|
||||
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
|
||||
self.analytics_api = analytics_api
|
||||
self._tracked_data = collections.defaultdict(list)
|
||||
self.looping_call_manager = self._setup_looping_calls()
|
||||
self.looping_tasks = {}
|
||||
self.context = context or self._make_context(
|
||||
system_info.get_platform(), conf.settings['wallet'])
|
||||
self.installation_id = installation_id or conf.settings.installation_id
|
||||
@@ -43,20 +44,20 @@ class Manager:
|
||||
return cls(api)
|
||||
|
||||
# Things We Track
|
||||
def send_new_download_start(self, download_id, name, claim_dict):
|
||||
self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
async def send_new_download_start(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("start", download_id, name, claim_dict)
|
||||
|
||||
def send_new_download_success(self, download_id, name, claim_dict):
|
||||
self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
async def send_new_download_success(self, download_id, name, claim_dict):
|
||||
await self._send_new_download_stats("success", download_id, name, claim_dict)
|
||||
|
||||
def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
async def send_new_download_fail(self, download_id, name, claim_dict, e):
|
||||
await self._send_new_download_stats("failure", download_id, name, claim_dict, {
|
||||
'name': type(e).__name__ if hasattr(type(e), "__name__") else str(type(e)),
|
||||
'message': str(e),
|
||||
})
|
||||
|
||||
def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
self.analytics_api.track({
|
||||
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
|
||||
await self.analytics_api.track({
|
||||
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
|
||||
'event': NEW_DOWNLOAD_STAT,
|
||||
'properties': self._event_properties({
|
||||
@@ -70,91 +71,81 @@ class Manager:
|
||||
'timestamp': utils.isonow(),
|
||||
})
|
||||
|
||||
def send_upnp_setup_success_fail(self, success, status):
|
||||
self.analytics_api.track(
|
||||
async def send_upnp_setup_success_fail(self, success, status):
|
||||
await self.analytics_api.track(
|
||||
self._event(UPNP_SETUP, {
|
||||
'success': success,
|
||||
'status': status,
|
||||
})
|
||||
)
|
||||
|
||||
def send_server_startup(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP))
|
||||
async def send_server_startup(self):
|
||||
await self.analytics_api.track(self._event(SERVER_STARTUP))
|
||||
|
||||
def send_server_startup_success(self):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
async def send_server_startup_success(self):
|
||||
await self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
|
||||
|
||||
def send_server_startup_error(self, message):
|
||||
self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
async def send_server_startup_error(self, message):
|
||||
await self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
|
||||
|
||||
def send_download_started(self, id_, name, claim_dict=None):
|
||||
self.analytics_api.track(
|
||||
async def send_download_started(self, id_, name, claim_dict=None):
|
||||
await self.analytics_api.track(
|
||||
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
|
||||
)
|
||||
|
||||
def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
async def send_download_errored(self, err, id_, name, claim_dict, report):
|
||||
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
|
||||
report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
await self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
|
||||
|
||||
def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
async def send_download_finished(self, id_, name, report, claim_dict=None):
|
||||
download_properties = self._download_properties(id_, name, claim_dict, report)
|
||||
self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
await self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
|
||||
|
||||
def send_claim_action(self, action):
|
||||
self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
async def send_claim_action(self, action):
|
||||
await self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
|
||||
|
||||
def send_new_channel(self):
|
||||
self.analytics_api.track(self._event(NEW_CHANNEL))
|
||||
async def send_new_channel(self):
|
||||
await self.analytics_api.track(self._event(NEW_CHANNEL))
|
||||
|
||||
def send_credits_sent(self):
|
||||
self.analytics_api.track(self._event(CREDITS_SENT))
|
||||
async def send_credits_sent(self):
|
||||
await self.analytics_api.track(self._event(CREDITS_SENT))
|
||||
|
||||
def _send_heartbeat(self):
|
||||
self.analytics_api.track(self._event(HEARTBEAT))
|
||||
async def _send_heartbeat(self):
|
||||
await self.analytics_api.track(self._event(HEARTBEAT))
|
||||
|
||||
def _update_tracked_metrics(self):
|
||||
async def _update_tracked_metrics(self):
|
||||
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
await self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
|
||||
|
||||
# Setup / Shutdown
|
||||
|
||||
def start(self):
|
||||
if not self.is_started:
|
||||
for name, _, interval in self._get_looping_calls():
|
||||
self.looping_call_manager.start(name, interval)
|
||||
for name, fn, secs in self._get_looping_calls():
|
||||
self.looping_tasks[name] = asyncio.create_task(looping_call(secs, fn))
|
||||
self.is_started = True
|
||||
log.info("Start")
|
||||
|
||||
def shutdown(self):
|
||||
self.looping_call_manager.shutdown()
|
||||
if self.is_started:
|
||||
try:
|
||||
for name, task in self.looping_tasks.items():
|
||||
if task:
|
||||
task.cancel()
|
||||
self.looping_tasks[name] = None
|
||||
log.info("Stopped analytics looping calls")
|
||||
self.is_started = False
|
||||
except Exception as e:
|
||||
log.exception('Got exception when trying to cancel tasks in analytics: ', exc_info=e)
|
||||
|
||||
def register_repeating_metric(self, event_name, value_generator, frequency=300):
|
||||
lcall = task.LoopingCall(self._send_repeating_metric, event_name, value_generator)
|
||||
self.looping_call_manager.register_looping_call(event_name, lcall)
|
||||
lcall.start(frequency)
|
||||
|
||||
def _get_looping_calls(self):
|
||||
def _get_looping_calls(self) -> list:
|
||||
return [
|
||||
('send_heartbeat', self._send_heartbeat, 300),
|
||||
('update_tracked_metrics', self._update_tracked_metrics, 600),
|
||||
]
|
||||
|
||||
def _setup_looping_calls(self):
|
||||
call_manager = looping_call_manager.LoopingCallManager()
|
||||
for name, fn, _ in self._get_looping_calls():
|
||||
call_manager.register_looping_call(name, task.LoopingCall(fn))
|
||||
return call_manager
|
||||
|
||||
def _send_repeating_metric(self, event_name, value_generator):
|
||||
result = value_generator()
|
||||
self._if_deferred(result, self._send_repeating_metric_value, event_name)
|
||||
|
||||
def _send_repeating_metric_value(self, result, event_name):
|
||||
should_send, value = result
|
||||
if should_send:
|
||||
self.analytics_api.track(self._metric_event(event_name, value))
|
||||
|
||||
def add_observation(self, metric, value):
|
||||
self._tracked_data[metric].append(value)
|
||||
|
||||
@@ -239,13 +230,6 @@ class Manager:
|
||||
context['os']['distro'] = platform['distro']
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def _if_deferred(maybe_deferred, callback, *args, **kwargs):
|
||||
if isinstance(maybe_deferred, defer.Deferred):
|
||||
maybe_deferred.addCallback(callback, *args, **kwargs)
|
||||
else:
|
||||
callback(maybe_deferred, *args, **kwargs)
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self, cookies, url, write_key, enabled):
|
||||
@@ -254,7 +238,7 @@ class Api:
|
||||
self._write_key = write_key
|
||||
self._enabled = enabled
|
||||
|
||||
def _post(self, endpoint, data):
|
||||
async def _post(self, endpoint, data):
|
||||
# there is an issue with a timing condition with keep-alive
|
||||
# that is best explained here: https://github.com/mikem23/keepalive-race
|
||||
#
|
||||
@@ -265,29 +249,28 @@ class Api:
|
||||
#
|
||||
# by forcing the connection to close, we will disable the keep-alive.
|
||||
|
||||
def update_cookies(response):
|
||||
self.cookies.update(response.cookies())
|
||||
return response
|
||||
|
||||
assert endpoint[0] == '/'
|
||||
headers = {b"Connection": b"close"}
|
||||
d = treq.post(self.url + endpoint, auth=(self._write_key, ''), json=data,
|
||||
headers=headers, cookies=self.cookies)
|
||||
d.addCallback(update_cookies)
|
||||
return d
|
||||
request_kwargs = {
|
||||
'method': 'POST',
|
||||
'url': self.url + endpoint,
|
||||
'headers': {'Connection': 'Close'},
|
||||
'auth': aiohttp.BasicAuth(self._write_key, ''),
|
||||
'json': data,
|
||||
'cookies': self.cookies
|
||||
}
|
||||
try:
|
||||
async with aiohttp.request(**request_kwargs) as response:
|
||||
self.cookies.update(response.cookies)
|
||||
except Exception as e:
|
||||
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
|
||||
|
||||
def track(self, event):
|
||||
async def track(self, event):
|
||||
"""Send a single tracking event"""
|
||||
if not self._enabled:
|
||||
return defer.succeed('Analytics disabled')
|
||||
|
||||
def _log_error(failure, event):
|
||||
log.warning('Failed to send track event. %s (%s)', failure.getTraceback(), str(event))
|
||||
return 'Analytics disabled'
|
||||
|
||||
log.debug('Sending track event: %s', event)
|
||||
d = self._post('/track', event)
|
||||
d.addErrback(_log_error, event)
|
||||
return d
|
||||
await self._post('/track', event)
|
||||
|
||||
@classmethod
|
||||
def new_instance(cls, enabled=None):
|
||||
|
||||
Whitespace-only changes.
@@ -1,34 +0,0 @@
|
||||
import logging
|
||||
from zope.interface import implementer
|
||||
from twisted.cred import portal, checkers, credentials, error as cred_error
|
||||
from twisted.internet import defer
|
||||
from twisted.web import resource
|
||||
from lbrynet.extras.daemon.auth.keyring import Keyring
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@implementer(portal.IRealm)
|
||||
class HttpPasswordRealm:
|
||||
def __init__(self, resource):
|
||||
self.resource = resource
|
||||
|
||||
def requestAvatar(self, avatarId, mind, *interfaces):
|
||||
log.debug("Processing request for %s", avatarId)
|
||||
if resource.IResource in interfaces:
|
||||
return (resource.IResource, self.resource, lambda: None)
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
@implementer(checkers.ICredentialsChecker)
|
||||
class PasswordChecker:
|
||||
credentialInterfaces = (credentials.IUsernamePassword,)
|
||||
|
||||
def __init__(self, keyring: Keyring):
|
||||
self.api_key = keyring.api_key
|
||||
|
||||
def requestAvatarId(self, creds):
|
||||
if creds.checkPassword(self.api_key.secret.encode()) and creds.username == self.api_key.name.encode():
|
||||
return defer.succeed(creds.username)
|
||||
log.warning('Incorrect username or password')
|
||||
return defer.fail(cred_error.UnauthorizedLogin('Incorrect username or password'))
|
||||
@@ -1,130 +0,0 @@
|
||||
import json
|
||||
import aiohttp
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.daemon.auth.keyring import Keyring, APIKey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
USER_AGENT = "AuthServiceProxy/0.1"
|
||||
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
|
||||
TWISTED_SESSION = "TWISTED_SESSION"
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
HTTP_TIMEOUT = 30
|
||||
|
||||
|
||||
class JSONRPCException(Exception):
|
||||
def __init__(self, rpc_error):
|
||||
super().__init__()
|
||||
self.error = rpc_error
|
||||
|
||||
|
||||
class UnAuthAPIClient:
|
||||
def __init__(self, host, port, session):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.session = session
|
||||
|
||||
def __getattr__(self, method):
|
||||
async def f(*args, **kwargs):
|
||||
return await self.call(method, [args, kwargs])
|
||||
|
||||
return f
|
||||
|
||||
@classmethod
|
||||
async def from_url(cls, url):
|
||||
url_fragment = urlparse(url)
|
||||
host = url_fragment.hostname
|
||||
port = url_fragment.port
|
||||
connector = aiohttp.TCPConnector(
|
||||
ssl=None if not conf.settings['use_https'] else Keyring.load_from_disk().ssl_context
|
||||
)
|
||||
session = aiohttp.ClientSession(connector=connector)
|
||||
return cls(host, port, session)
|
||||
|
||||
async def call(self, method, params=None):
|
||||
message = {'method': method, 'params': params}
|
||||
async with self.session.get(conf.settings.get_api_connection_string(), json=message) as resp:
|
||||
return await resp.json()
|
||||
|
||||
|
||||
class AuthAPIClient:
|
||||
def __init__(self, key, session, cookies, url, login_url):
|
||||
self.session = session
|
||||
self.__api_key = key
|
||||
self.__login_url = login_url
|
||||
self.__id_count = 0
|
||||
self.__url = url
|
||||
self.__cookies = cookies
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name.startswith('__') and name.endswith('__'):
|
||||
raise AttributeError(name)
|
||||
|
||||
def f(*args, **kwargs):
|
||||
return self.call(name, [args, kwargs])
|
||||
|
||||
return f
|
||||
|
||||
async def call(self, method, params=None):
|
||||
params = params or {}
|
||||
self.__id_count += 1
|
||||
|
||||
pre_auth_post_data = {
|
||||
'version': '2',
|
||||
'method': method,
|
||||
'params': params,
|
||||
'id': self.__id_count
|
||||
}
|
||||
to_auth = json.dumps(pre_auth_post_data, sort_keys=True)
|
||||
auth_msg = self.__api_key.get_hmac(to_auth).decode()
|
||||
pre_auth_post_data.update({'hmac': auth_msg})
|
||||
post_data = json.dumps(pre_auth_post_data)
|
||||
|
||||
headers = {
|
||||
'Host': self.__url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'
|
||||
}
|
||||
|
||||
async with self.session.post(self.__login_url, data=post_data, headers=headers) as resp:
|
||||
if resp is None:
|
||||
raise JSONRPCException({'code': -342, 'message': 'missing HTTP response from server'})
|
||||
resp.raise_for_status()
|
||||
|
||||
next_secret = resp.headers.get(LBRY_SECRET, False)
|
||||
if next_secret:
|
||||
self.__api_key.secret = next_secret
|
||||
|
||||
return await resp.json()
|
||||
|
||||
@classmethod
|
||||
async def get_client(cls, key_name=None):
|
||||
api_key_name = key_name or "api"
|
||||
keyring = Keyring.load_from_disk()
|
||||
|
||||
api_key = keyring.api_key
|
||||
login_url = conf.settings.get_api_connection_string(api_key_name, api_key.secret)
|
||||
url = urlparse(login_url)
|
||||
|
||||
headers = {
|
||||
'Host': url.hostname,
|
||||
'User-Agent': USER_AGENT,
|
||||
'Content-type': 'application/json'
|
||||
}
|
||||
connector = aiohttp.TCPConnector(ssl=None if not conf.settings['use_https'] else keyring.ssl_context)
|
||||
session = aiohttp.ClientSession(connector=connector)
|
||||
|
||||
async with session.post(login_url, headers=headers) as r:
|
||||
cookies = r.cookies
|
||||
uid = cookies.get(TWISTED_SECURE_SESSION if conf.settings['use_https'] else TWISTED_SESSION).value
|
||||
api_key = APIKey.create(seed=uid.encode())
|
||||
return cls(api_key, session, cookies, url, login_url)
|
||||
|
||||
|
||||
class LBRYAPIClient:
|
||||
@staticmethod
|
||||
def get_client():
|
||||
return AuthAPIClient.get_client() if conf.settings['use_auth_http'] else \
|
||||
UnAuthAPIClient.from_url(conf.settings.get_api_connection_string())
|
||||
@@ -1,50 +0,0 @@
|
||||
import logging
|
||||
|
||||
from twisted.web import server, guard, resource
|
||||
from twisted.cred import portal
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.daemon.auth.auth import PasswordChecker, HttpPasswordRealm
|
||||
from lbrynet.extras.daemon.auth.keyring import Keyring
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HTTPJSONRPCFactory(server.Site):
|
||||
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
|
||||
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
|
||||
self.use_ssl = False
|
||||
|
||||
|
||||
class HTTPSJSONRPCFactory(server.Site):
|
||||
def __init__(self, resource, keyring, requestFactory=None, *args, **kwargs):
|
||||
super().__init__(resource, requestFactory=requestFactory, *args, **kwargs)
|
||||
self.options = keyring.private_certificate.options()
|
||||
self.use_ssl = True
|
||||
|
||||
|
||||
class AuthJSONRPCResource(resource.Resource):
|
||||
def __init__(self, protocol):
|
||||
resource.Resource.__init__(self)
|
||||
self.putChild(b"", protocol)
|
||||
self.putChild(conf.settings['API_ADDRESS'].encode(), protocol)
|
||||
|
||||
def getChild(self, name, request):
|
||||
request.setHeader('cache-control', 'no-cache, no-store, must-revalidate')
|
||||
request.setHeader('expires', '0')
|
||||
return self if name == '' else resource.Resource.getChild(self, name, request)
|
||||
|
||||
def getServerFactory(self, keyring: Keyring, use_authentication: bool, use_https: bool) -> server.Site:
|
||||
factory_class = HTTPSJSONRPCFactory if use_https else HTTPJSONRPCFactory
|
||||
if use_authentication:
|
||||
log.info("Using authenticated API")
|
||||
checker = PasswordChecker(keyring)
|
||||
realm = HttpPasswordRealm(self)
|
||||
portal_to_realm = portal.Portal(realm, [checker, ])
|
||||
root = guard.HTTPAuthSessionWrapper(
|
||||
portal_to_realm, [guard.BasicCredentialFactory(b'Login to lbrynet api'), ]
|
||||
)
|
||||
else:
|
||||
log.info("Using non-authenticated API")
|
||||
root = self
|
||||
return factory_class(root, keyring)
|
||||
@@ -1,131 +0,0 @@
|
||||
import os
|
||||
import datetime
|
||||
import hmac
|
||||
import hashlib
|
||||
import base58
|
||||
import logging
|
||||
from OpenSSL.crypto import FILETYPE_PEM
|
||||
from ssl import create_default_context, SSLContext
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.x509.name import NameOID, NameAttribute
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from twisted.internet import ssl
|
||||
|
||||
from lbrynet import conf
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def sha(x: bytes) -> str:
|
||||
h = hashlib.sha256(x).digest()
|
||||
return base58.b58encode(h).decode()
|
||||
|
||||
|
||||
def generate_key(x: bytes = None) -> str:
|
||||
if not x:
|
||||
return sha(os.urandom(256))
|
||||
else:
|
||||
return sha(x)
|
||||
|
||||
|
||||
class APIKey:
|
||||
def __init__(self, secret: str, name: str):
|
||||
self.secret = secret
|
||||
self.name = name
|
||||
|
||||
@classmethod
|
||||
def create(cls, seed=None, name=None):
|
||||
secret = generate_key(seed)
|
||||
return APIKey(secret, name)
|
||||
|
||||
def _raw_key(self) -> str:
|
||||
return base58.b58decode(self.secret)
|
||||
|
||||
def get_hmac(self, message) -> str:
|
||||
decoded_key = self._raw_key()
|
||||
signature = hmac.new(decoded_key, message.encode(), hashlib.sha256)
|
||||
return base58.b58encode(signature.digest())
|
||||
|
||||
def compare_hmac(self, message, token) -> bool:
|
||||
decoded_token = base58.b58decode(token)
|
||||
target = base58.b58decode(self.get_hmac(message))
|
||||
|
||||
try:
|
||||
if len(decoded_token) != len(target):
|
||||
return False
|
||||
return hmac.compare_digest(decoded_token, target)
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
class Keyring:
|
||||
encoding = serialization.Encoding.PEM
|
||||
filetype = FILETYPE_PEM
|
||||
|
||||
def __init__(self, api_key: APIKey, public_certificate: str, private_certificate: ssl.PrivateCertificate = None):
|
||||
self.api_key: APIKey = api_key
|
||||
self.public_certificate: str = public_certificate
|
||||
self.private_certificate: (ssl.PrivateCertificate or None) = private_certificate
|
||||
self.ssl_context: SSLContext = create_default_context(cadata=self.public_certificate)
|
||||
|
||||
@classmethod
|
||||
def load_from_disk(cls):
|
||||
api_key_path = os.path.join(conf.settings.data_dir, 'auth_token')
|
||||
api_ssl_cert_path = os.path.join(conf.settings.data_dir, 'api_ssl_cert.pem')
|
||||
if not os.path.isfile(api_key_path) or not os.path.isfile(api_ssl_cert_path):
|
||||
return
|
||||
with open(api_key_path, 'rb') as f:
|
||||
api_key = APIKey(f.read().decode(), "api")
|
||||
with open(api_ssl_cert_path, 'rb') as f:
|
||||
public_cert = f.read().decode()
|
||||
return cls(api_key, public_cert)
|
||||
|
||||
@classmethod
|
||||
def generate_and_save(cls):
|
||||
dns = conf.settings['api_host']
|
||||
private_key = rsa.generate_private_key(
|
||||
public_exponent=65537,
|
||||
key_size=4096,
|
||||
backend=default_backend()
|
||||
)
|
||||
subject = issuer = x509.Name([
|
||||
NameAttribute(NameOID.COUNTRY_NAME, "US"),
|
||||
NameAttribute(NameOID.ORGANIZATION_NAME, "LBRY"),
|
||||
NameAttribute(NameOID.COMMON_NAME, "LBRY API"),
|
||||
])
|
||||
alternative_name = x509.SubjectAlternativeName([x509.DNSName(dns)])
|
||||
certificate = x509.CertificateBuilder(
|
||||
subject_name=subject,
|
||||
issuer_name=issuer,
|
||||
public_key=private_key.public_key(),
|
||||
serial_number=x509.random_serial_number(),
|
||||
not_valid_before=datetime.datetime.utcnow(),
|
||||
not_valid_after=datetime.datetime.utcnow() + datetime.timedelta(days=365),
|
||||
extensions=[x509.Extension(oid=alternative_name.oid, critical=False, value=alternative_name)]
|
||||
).sign(private_key, hashes.SHA256(), default_backend())
|
||||
public_certificate = certificate.public_bytes(cls.encoding).decode()
|
||||
private_certificate = ssl.PrivateCertificate.load(
|
||||
public_certificate,
|
||||
ssl.KeyPair.load(
|
||||
private_key.private_bytes(
|
||||
encoding=cls.encoding,
|
||||
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
encryption_algorithm=serialization.NoEncryption()
|
||||
).decode(),
|
||||
cls.filetype
|
||||
),
|
||||
cls.filetype
|
||||
)
|
||||
|
||||
auth_token = APIKey.create(seed=None, name="api")
|
||||
|
||||
with open(os.path.join(conf.settings.data_dir, 'auth_token'), 'wb') as f:
|
||||
f.write(auth_token.secret.encode())
|
||||
|
||||
with open(os.path.join(conf.settings.data_dir, 'api_ssl_cert.pem'), 'wb') as f:
|
||||
f.write(public_certificate.encode())
|
||||
|
||||
return cls(auth_token, public_certificate, private_certificate)
|
||||
@@ -1,611 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from urllib import parse as urlparse
|
||||
import json
|
||||
import inspect
|
||||
import signal
|
||||
from functools import wraps
|
||||
from twisted.web import server
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet.error import ConnectionDone, ConnectionLost
|
||||
from txjsonrpc import jsonrpclib
|
||||
from traceback import format_exc
|
||||
|
||||
from lbrynet import conf, utils
|
||||
from lbrynet.extras.daemon import analytics
|
||||
from lbrynet.p2p.Error import InvalidAuthenticationToken
|
||||
from lbrynet.p2p.Error import ComponentsNotStarted, ComponentStartConditionNotMet
|
||||
from lbrynet.extras.looping_call_manager import LoopingCallManager
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.auth.keyring import APIKey, Keyring
|
||||
from lbrynet.extras.daemon.auth.undecorated import undecorated
|
||||
from lbrynet.extras.daemon.auth.factory import AuthJSONRPCResource
|
||||
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
EMPTY_PARAMS = [{}]
|
||||
LBRY_SECRET = "LBRY_SECRET"
|
||||
|
||||
|
||||
class JSONRPCError:
|
||||
# http://www.jsonrpc.org/specification#error_object
|
||||
CODE_PARSE_ERROR = -32700 # Invalid JSON. Error while parsing the JSON text.
|
||||
CODE_INVALID_REQUEST = -32600 # The JSON sent is not a valid Request object.
|
||||
CODE_METHOD_NOT_FOUND = -32601 # The method does not exist / is not available.
|
||||
CODE_INVALID_PARAMS = -32602 # Invalid method parameter(s).
|
||||
CODE_INTERNAL_ERROR = -32603 # Internal JSON-RPC error (I think this is like a 500?)
|
||||
CODE_APPLICATION_ERROR = -32500 # Generic error with our app??
|
||||
CODE_AUTHENTICATION_ERROR = -32501 # Authentication failed
|
||||
|
||||
MESSAGES = {
|
||||
CODE_PARSE_ERROR: "Parse Error. Data is not valid JSON.",
|
||||
CODE_INVALID_REQUEST: "JSON data is not a valid Request",
|
||||
CODE_METHOD_NOT_FOUND: "Method Not Found",
|
||||
CODE_INVALID_PARAMS: "Invalid Params",
|
||||
CODE_INTERNAL_ERROR: "Internal Error",
|
||||
CODE_AUTHENTICATION_ERROR: "Authentication Failed",
|
||||
}
|
||||
|
||||
HTTP_CODES = {
|
||||
CODE_INVALID_REQUEST: 400,
|
||||
CODE_PARSE_ERROR: 400,
|
||||
CODE_INVALID_PARAMS: 400,
|
||||
CODE_METHOD_NOT_FOUND: 404,
|
||||
CODE_INTERNAL_ERROR: 500,
|
||||
CODE_APPLICATION_ERROR: 500,
|
||||
CODE_AUTHENTICATION_ERROR: 401,
|
||||
}
|
||||
|
||||
def __init__(self, message, code=CODE_APPLICATION_ERROR, traceback=None, data=None):
|
||||
assert isinstance(code, int), "'code' must be an int"
|
||||
assert (data is None or isinstance(data, dict)), "'data' must be None or a dict"
|
||||
self.code = code
|
||||
if message is None:
|
||||
message = self.MESSAGES[code] if code in self.MESSAGES else "API Error"
|
||||
self.message = message
|
||||
self.data = {} if data is None else data
|
||||
self.traceback = []
|
||||
if traceback is not None:
|
||||
trace_lines = traceback.split("\n")
|
||||
for i, t in enumerate(trace_lines):
|
||||
if "--- <exception caught here> ---" in t:
|
||||
if len(trace_lines) > i + 1:
|
||||
self.traceback = [j for j in trace_lines[i+1:] if j]
|
||||
break
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'code': self.code,
|
||||
'message': self.message,
|
||||
'data': self.traceback
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def create_from_exception(cls, message, code=CODE_APPLICATION_ERROR, traceback=None):
|
||||
return cls(message, code=code, traceback=traceback)
|
||||
|
||||
|
||||
class UnknownAPIMethodError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def trap(err, *to_trap):
|
||||
err.trap(*to_trap)
|
||||
|
||||
|
||||
def jsonrpc_dumps_pretty(obj, **kwargs):
|
||||
try:
|
||||
id_ = kwargs.pop("id")
|
||||
except KeyError:
|
||||
id_ = None
|
||||
|
||||
if isinstance(obj, JSONRPCError):
|
||||
data = {"jsonrpc": "2.0", "error": obj.to_dict(), "id": id_}
|
||||
else:
|
||||
data = {"jsonrpc": "2.0", "result": obj, "id": id_}
|
||||
|
||||
return json.dumps(data, cls=JSONResponseEncoder, sort_keys=True, indent=2, **kwargs) + "\n"
|
||||
|
||||
|
||||
class JSONRPCServerType(type):
|
||||
def __new__(mcs, name, bases, newattrs):
|
||||
klass = type.__new__(mcs, name, bases, newattrs)
|
||||
klass.callable_methods = {}
|
||||
klass.deprecated_methods = {}
|
||||
|
||||
for methodname in dir(klass):
|
||||
if methodname.startswith("jsonrpc_"):
|
||||
method = getattr(klass, methodname)
|
||||
if not hasattr(method, '_deprecated'):
|
||||
klass.callable_methods.update({methodname.split("jsonrpc_")[1]: method})
|
||||
else:
|
||||
klass.deprecated_methods.update({methodname.split("jsonrpc_")[1]: method})
|
||||
return klass
|
||||
|
||||
|
||||
class AuthorizedBase(metaclass=JSONRPCServerType):
|
||||
|
||||
@staticmethod
|
||||
def deprecated(new_command=None):
|
||||
def _deprecated_wrapper(f):
|
||||
f.new_command = new_command
|
||||
f._deprecated = True
|
||||
return f
|
||||
return _deprecated_wrapper
|
||||
|
||||
@staticmethod
|
||||
def requires(*components, **conditions):
|
||||
if conditions and ["conditions"] != list(conditions.keys()):
|
||||
raise SyntaxError("invalid conditions argument")
|
||||
condition_names = conditions.get("conditions", [])
|
||||
|
||||
def _wrap(fn):
|
||||
@wraps(fn)
|
||||
def _inner(*args, **kwargs):
|
||||
component_manager = args[0].component_manager
|
||||
for condition_name in condition_names:
|
||||
condition_result, err_msg = component_manager.evaluate_condition(condition_name)
|
||||
if not condition_result:
|
||||
raise ComponentStartConditionNotMet(err_msg)
|
||||
if not component_manager.all_components_running(*components):
|
||||
raise ComponentsNotStarted("the following required components have not yet started: "
|
||||
"%s" % json.dumps(components))
|
||||
return fn(*args, **kwargs)
|
||||
return _inner
|
||||
return _wrap
|
||||
|
||||
|
||||
class AuthJSONRPCServer(AuthorizedBase):
|
||||
"""
|
||||
Authorized JSONRPC server used as the base class for the LBRY API
|
||||
|
||||
API methods are named with a leading "jsonrpc_"
|
||||
|
||||
Attributes:
|
||||
sessions (dict): (dict): {<session id>: <lbrynet.daemon.auth.util.APIKey>}
|
||||
callable_methods (dict): {<api method name>: <api method>}
|
||||
|
||||
Authentication:
|
||||
If use_authentication is true, basic HTTP and HMAC authentication will be used for all requests and the
|
||||
service url will require a username and password.
|
||||
|
||||
To start an authenticated session a client sends an HTTP POST to <user>:<password>@<api host>:<api port>.
|
||||
If accepted, the server replies with a TWISTED_SESSION cookie containing a session id and the message "OK".
|
||||
The client initializes their shared secret for hmac to be the b64 encoded sha256 of their session id.
|
||||
|
||||
To send an authenticated request a client sends an HTTP POST to the auth api url with the TWISTED_SESSION
|
||||
cookie and includes a hmac token in the message using the previously set shared secret. If the token is valid
|
||||
the server will randomize the shared secret and return the new value under the LBRY_SECRET header, which the
|
||||
client uses to generate the token for their next request.
|
||||
"""
|
||||
#implements(resource.IResource)
|
||||
|
||||
isLeaf = True
|
||||
allowed_during_startup = []
|
||||
component_attributes = {}
|
||||
|
||||
def __init__(self, analytics_manager=None, component_manager=None, use_authentication=None, use_https=None,
|
||||
to_skip=None, looping_calls=None, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.analytics_manager = analytics_manager or analytics.Manager.new_instance()
|
||||
self.component_manager = component_manager or ComponentManager(
|
||||
analytics_manager=self.analytics_manager,
|
||||
skip_components=to_skip or [],
|
||||
reactor=reactor
|
||||
)
|
||||
self.looping_call_manager = LoopingCallManager({n: lc for n, (lc, t) in (looping_calls or {}).items()})
|
||||
self._looping_call_times = {n: t for n, (lc, t) in (looping_calls or {}).items()}
|
||||
self._use_authentication = use_authentication or conf.settings['use_auth_http']
|
||||
self._use_https = use_https or conf.settings['use_https']
|
||||
self.listening_port = None
|
||||
self._component_setup_deferred = None
|
||||
self.announced_startup = False
|
||||
self.sessions = {}
|
||||
self.server = None
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start_listening(self):
|
||||
from twisted.internet import reactor, error as tx_error
|
||||
|
||||
try:
|
||||
self.server = self.get_server_factory()
|
||||
if self.server.use_ssl:
|
||||
log.info("Using SSL")
|
||||
self.listening_port = reactor.listenSSL(
|
||||
conf.settings['api_port'], self.server, self.server.options, interface=conf.settings['api_host']
|
||||
)
|
||||
else:
|
||||
log.info("Not using SSL")
|
||||
self.listening_port = reactor.listenTCP(
|
||||
conf.settings['api_port'], self.server, interface=conf.settings['api_host']
|
||||
)
|
||||
log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port'])
|
||||
yield self.setup()
|
||||
self.analytics_manager.send_server_startup_success()
|
||||
except tx_error.CannotListenError:
|
||||
log.error('lbrynet API failed to bind TCP %s:%i for listening. Daemon is already running or this port is '
|
||||
'already in use by another application.', conf.settings['api_host'], conf.settings['api_port'])
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
except defer.CancelledError:
|
||||
log.info("shutting down before finished starting")
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
except Exception as err:
|
||||
self.analytics_manager.send_server_startup_error(str(err))
|
||||
log.exception('Failed to start lbrynet-daemon')
|
||||
reactor.fireSystemEvent("shutdown")
|
||||
|
||||
def setup(self):
|
||||
from twisted.internet import reactor
|
||||
|
||||
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
|
||||
if not self.analytics_manager.is_started:
|
||||
self.analytics_manager.start()
|
||||
self.analytics_manager.send_server_startup()
|
||||
for lc_name, lc_time in self._looping_call_times.items():
|
||||
self.looping_call_manager.start(lc_name, lc_time)
|
||||
|
||||
def update_attribute(setup_result, component):
|
||||
setattr(self, self.component_attributes[component.component_name], component.component)
|
||||
|
||||
kwargs = {component: update_attribute for component in self.component_attributes.keys()}
|
||||
self._component_setup_deferred = self.component_manager.setup(**kwargs)
|
||||
return self._component_setup_deferred
|
||||
|
||||
@staticmethod
|
||||
def _already_shutting_down(sig_num, frame):
|
||||
log.info("Already shutting down")
|
||||
|
||||
def _shutdown(self):
|
||||
# ignore INT/TERM signals once shutdown has started
|
||||
signal.signal(signal.SIGINT, self._already_shutting_down)
|
||||
signal.signal(signal.SIGTERM, self._already_shutting_down)
|
||||
if self.listening_port:
|
||||
self.listening_port.stopListening()
|
||||
self.looping_call_manager.shutdown()
|
||||
if self.server is not None:
|
||||
for session in list(self.server.sessions.values()):
|
||||
session.expire()
|
||||
if self.analytics_manager:
|
||||
self.analytics_manager.shutdown()
|
||||
try:
|
||||
self._component_setup_deferred.cancel()
|
||||
except (AttributeError, defer.CancelledError):
|
||||
pass
|
||||
if self.component_manager is not None:
|
||||
d = self.component_manager.stop()
|
||||
d.addErrback(log.fail(), 'Failure while shutting down')
|
||||
else:
|
||||
d = defer.succeed(None)
|
||||
return d
|
||||
|
||||
def get_server_factory(self):
|
||||
keyring = None
|
||||
if self._use_authentication or self._use_https:
|
||||
keyring = Keyring.generate_and_save()
|
||||
return AuthJSONRPCResource(self).getServerFactory(keyring, self._use_authentication, self._use_https)
|
||||
|
||||
def _set_headers(self, request, data, update_secret=False):
|
||||
if conf.settings['allowed_origin']:
|
||||
request.setHeader("Access-Control-Allow-Origin", conf.settings['allowed_origin'])
|
||||
request.setHeader("Content-Type", "application/json")
|
||||
request.setHeader("Accept", "application/json-rpc")
|
||||
request.setHeader("Content-Length", str(len(data)))
|
||||
if update_secret:
|
||||
session_id = request.getSession().uid
|
||||
request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret)
|
||||
|
||||
@staticmethod
|
||||
def _render_message(request, message: str):
|
||||
request.write(message.encode())
|
||||
request.finish()
|
||||
|
||||
def _render_error(self, failure, request, id_):
|
||||
if isinstance(failure, JSONRPCError):
|
||||
error = failure
|
||||
elif isinstance(failure, Failure):
|
||||
# maybe failure is JSONRPCError wrapped in a twisted Failure
|
||||
error = failure.check(JSONRPCError)
|
||||
if error is None:
|
||||
# maybe its a twisted Failure with another type of error
|
||||
if hasattr(failure.type, "code"):
|
||||
error_code = failure.type.code
|
||||
else:
|
||||
error_code = JSONRPCError.CODE_APPLICATION_ERROR
|
||||
error = JSONRPCError.create_from_exception(
|
||||
failure.getErrorMessage() or failure.type.__name__,
|
||||
code=error_code,
|
||||
traceback=failure.getTraceback()
|
||||
)
|
||||
if not failure.check(ComponentsNotStarted, ComponentStartConditionNotMet):
|
||||
log.warning("error processing api request: %s\ntraceback: %s", error.message,
|
||||
"\n".join(error.traceback))
|
||||
else:
|
||||
# last resort, just cast it as a string
|
||||
error = JSONRPCError(str(failure))
|
||||
|
||||
response_content = jsonrpc_dumps_pretty(error, id=id_, ledger=self.ledger)
|
||||
self._set_headers(request, response_content)
|
||||
request.setResponseCode(200)
|
||||
self._render_message(request, response_content)
|
||||
|
||||
@staticmethod
|
||||
def _handle_dropped_request(result, d, function_name):
|
||||
if not d.called:
|
||||
log.warning("Cancelling dropped api request %s", function_name)
|
||||
d.cancel()
|
||||
|
||||
def render(self, request):
|
||||
try:
|
||||
return self._render(request)
|
||||
except BaseException as e:
|
||||
log.error(e)
|
||||
error = JSONRPCError.create_from_exception(str(e), traceback=format_exc())
|
||||
self._render_error(error, request, None)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def _render(self, request):
|
||||
time_in = utils.now()
|
||||
# if not self._check_headers(request):
|
||||
# self._render_error(Failure(InvalidHeaderError()), request, None)
|
||||
# return server.NOT_DONE_YET
|
||||
session = request.getSession()
|
||||
session_id = session.uid
|
||||
finished_deferred = request.notifyFinish()
|
||||
|
||||
if self._use_authentication:
|
||||
# if this is a new session, send a new secret and set the expiration
|
||||
# otherwise, session.touch()
|
||||
if self._initialize_session(session_id):
|
||||
def expire_session():
|
||||
self._unregister_user_session(session_id)
|
||||
|
||||
session.notifyOnExpire(expire_session)
|
||||
message = "OK"
|
||||
request.setResponseCode(200)
|
||||
self._set_headers(request, message, True)
|
||||
self._render_message(request, message)
|
||||
return server.NOT_DONE_YET
|
||||
else:
|
||||
session.touch()
|
||||
|
||||
request.content.seek(0, 0)
|
||||
content = request.content.read().decode()
|
||||
try:
|
||||
parsed = jsonrpclib.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
log.warning("Unable to decode request json")
|
||||
self._render_error(JSONRPCError(None, code=JSONRPCError.CODE_PARSE_ERROR), request, None)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
request_id = None
|
||||
try:
|
||||
function_name = parsed.get('method')
|
||||
args = parsed.get('params', {})
|
||||
request_id = parsed.get('id', None)
|
||||
token = parsed.pop('hmac', None)
|
||||
except AttributeError as err:
|
||||
log.warning(err)
|
||||
self._render_error(
|
||||
JSONRPCError(None, code=JSONRPCError.CODE_INVALID_REQUEST), request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
reply_with_next_secret = False
|
||||
if self._use_authentication:
|
||||
try:
|
||||
self._verify_token(session_id, parsed, token)
|
||||
except InvalidAuthenticationToken as err:
|
||||
log.warning("API validation failed")
|
||||
self._render_error(
|
||||
JSONRPCError.create_from_exception(
|
||||
str(err),
|
||||
code=JSONRPCError.CODE_AUTHENTICATION_ERROR,
|
||||
traceback=format_exc()
|
||||
),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
request.addCookie("TWISTED_SESSION", session_id)
|
||||
self._update_session_secret(session_id)
|
||||
reply_with_next_secret = True
|
||||
|
||||
try:
|
||||
fn = self._get_jsonrpc_method(function_name)
|
||||
except UnknownAPIMethodError as err:
|
||||
log.warning('Failed to get function %s: %s', function_name, err)
|
||||
self._render_error(
|
||||
JSONRPCError(None, code=JSONRPCError.CODE_METHOD_NOT_FOUND),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
if args in (EMPTY_PARAMS, []):
|
||||
_args, _kwargs = (), {}
|
||||
elif isinstance(args, dict):
|
||||
_args, _kwargs = (), args
|
||||
elif len(args) == 1 and isinstance(args[0], dict):
|
||||
# TODO: this is for backwards compatibility. Remove this once API and UI are updated
|
||||
# TODO: also delete EMPTY_PARAMS then
|
||||
_args, _kwargs = (), args[0]
|
||||
elif len(args) == 2 and isinstance(args[0], list) and isinstance(args[1], dict):
|
||||
_args, _kwargs = args
|
||||
else:
|
||||
raise ValueError('invalid args format')
|
||||
|
||||
params_error, erroneous_params = self._check_params(fn, _args, _kwargs)
|
||||
if params_error is not None:
|
||||
params_error_message = '{} for {} command: {}'.format(
|
||||
params_error, function_name, ', '.join(erroneous_params)
|
||||
)
|
||||
log.warning(params_error_message)
|
||||
self._render_error(
|
||||
JSONRPCError(params_error_message, code=JSONRPCError.CODE_INVALID_PARAMS),
|
||||
request, request_id
|
||||
)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
try:
|
||||
result = fn(self, *_args, **_kwargs)
|
||||
if isinstance(result, Deferred):
|
||||
d = result
|
||||
elif isinstance(result, Failure):
|
||||
d = defer.fail(result)
|
||||
elif asyncio.iscoroutine(result):
|
||||
d = Deferred.fromFuture(asyncio.ensure_future(result))
|
||||
else:
|
||||
d = defer.succeed(result)
|
||||
except:
|
||||
d = defer.fail(Failure(captureVars=Deferred.debug))
|
||||
|
||||
# finished_deferred will callback when the request is finished
|
||||
# and errback if something went wrong. If the errback is
|
||||
# called, cancel the deferred stack. This is to prevent
|
||||
# request.finish() from being called on a closed request.
|
||||
finished_deferred.addErrback(self._handle_dropped_request, d, function_name)
|
||||
|
||||
d.addCallback(self._callback_render, request, request_id, reply_with_next_secret)
|
||||
d.addErrback(trap, ConnectionDone, ConnectionLost, defer.CancelledError)
|
||||
d.addErrback(self._render_error, request, request_id)
|
||||
d.addBoth(lambda _: log.debug("%s took %f",
|
||||
function_name,
|
||||
(utils.now() - time_in).total_seconds()))
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
def _register_user_session(self, session_id):
|
||||
"""
|
||||
Add or update a HMAC secret for a session
|
||||
|
||||
@param session_id:
|
||||
@return: secret
|
||||
"""
|
||||
log.info("Started new api session")
|
||||
token = APIKey.create(seed=session_id)
|
||||
self.sessions.update({session_id: token})
|
||||
|
||||
def _unregister_user_session(self, session_id):
|
||||
log.info("Unregister API session")
|
||||
del self.sessions[session_id]
|
||||
|
||||
def _check_headers(self, request):
|
||||
return (
|
||||
self._check_header_source(request, 'Origin') and
|
||||
self._check_header_source(request, 'Referer')
|
||||
)
|
||||
|
||||
def _check_header_source(self, request, header):
|
||||
"""Check if the source of the request is allowed based on the header value."""
|
||||
source = request.getHeader(header)
|
||||
if not self._check_source_of_request(source):
|
||||
log.warning("Attempted api call from invalid %s: %s", header, source)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _check_source_of_request(self, source):
|
||||
if source is None:
|
||||
return True
|
||||
if conf.settings['api_host'] == '0.0.0.0':
|
||||
return True
|
||||
server, port = self.get_server_port(source)
|
||||
return self._check_server_port(server, port)
|
||||
|
||||
def _check_server_port(self, server, port):
|
||||
api = (conf.settings['api_host'], conf.settings['api_port'])
|
||||
return (server, port) == api or self._is_from_allowed_origin(server, port)
|
||||
|
||||
def _is_from_allowed_origin(self, server, port):
|
||||
allowed_origin = conf.settings['allowed_origin']
|
||||
if not allowed_origin:
|
||||
return False
|
||||
if allowed_origin == '*':
|
||||
return True
|
||||
allowed_server, allowed_port = self.get_server_port(allowed_origin)
|
||||
return (allowed_server, allowed_port) == (server, port)
|
||||
|
||||
def get_server_port(self, origin):
|
||||
parsed = urlparse.urlparse(origin)
|
||||
server_port = parsed.netloc.split(':')
|
||||
assert len(server_port) <= 2
|
||||
if len(server_port) == 2:
|
||||
return server_port[0], int(server_port[1])
|
||||
else:
|
||||
return server_port[0], 80
|
||||
|
||||
def _verify_method_is_callable(self, function_path):
|
||||
if function_path not in self.callable_methods:
|
||||
raise UnknownAPIMethodError(function_path)
|
||||
|
||||
def _get_jsonrpc_method(self, function_path):
|
||||
if function_path in self.deprecated_methods:
|
||||
new_command = self.deprecated_methods[function_path].new_command
|
||||
log.warning('API function \"%s\" is deprecated, please update to use \"%s\"',
|
||||
function_path, new_command)
|
||||
function_path = new_command
|
||||
self._verify_method_is_callable(function_path)
|
||||
return self.callable_methods.get(function_path)
|
||||
|
||||
@staticmethod
|
||||
def _check_params(function, args_tup, args_dict):
|
||||
argspec = inspect.getfullargspec(undecorated(function))
|
||||
num_optional_params = 0 if argspec.defaults is None else len(argspec.defaults)
|
||||
|
||||
duplicate_params = [
|
||||
duplicate_param
|
||||
for duplicate_param in argspec.args[1:len(args_tup) + 1]
|
||||
if duplicate_param in args_dict
|
||||
]
|
||||
|
||||
if duplicate_params:
|
||||
return 'Duplicate parameters', duplicate_params
|
||||
|
||||
missing_required_params = [
|
||||
required_param
|
||||
for required_param in argspec.args[len(args_tup)+1:-num_optional_params]
|
||||
if required_param not in args_dict
|
||||
]
|
||||
if len(missing_required_params):
|
||||
return 'Missing required parameters', missing_required_params
|
||||
|
||||
extraneous_params = [] if argspec.varkw is not None else [
|
||||
extra_param
|
||||
for extra_param in args_dict
|
||||
if extra_param not in argspec.args[1:]
|
||||
]
|
||||
if len(extraneous_params):
|
||||
return 'Extraneous parameters', extraneous_params
|
||||
|
||||
return None, None
|
||||
|
||||
def _initialize_session(self, session_id):
|
||||
if not self.sessions.get(session_id):
|
||||
self._register_user_session(session_id)
|
||||
return True
|
||||
return False
|
||||
|
||||
def _verify_token(self, session_id, message, token):
|
||||
if token is None:
|
||||
raise InvalidAuthenticationToken('Authentication token not found')
|
||||
to_auth = json.dumps(message, sort_keys=True)
|
||||
api_key = self.sessions.get(session_id)
|
||||
if not api_key.compare_hmac(to_auth, token):
|
||||
raise InvalidAuthenticationToken('Invalid authentication token')
|
||||
|
||||
def _update_session_secret(self, session_id):
|
||||
self.sessions.update({session_id: APIKey.create(name=session_id)})
|
||||
|
||||
def _callback_render(self, result, request, id_, auth_required=False):
|
||||
try:
|
||||
message = jsonrpc_dumps_pretty(result, id=id_, ledger=self.ledger)
|
||||
request.setResponseCode(200)
|
||||
self._set_headers(request, message, auth_required)
|
||||
self._render_message(request, message)
|
||||
except Exception as err:
|
||||
log.exception("Failed to render API response: %s", result)
|
||||
self._render_error(err, request, id_)
|
||||
|
||||
@staticmethod
|
||||
def _render_response(result):
|
||||
return defer.succeed(result)
|
||||
@@ -75,7 +75,7 @@ class JSONResponseEncoder(JSONEncoder):
|
||||
output['channel_name'] = txo.channel.claim_name
|
||||
try:
|
||||
output['valid_signature'] = claim.validate_signature(
|
||||
txo.get_address(self.ledger), txo.channel.claim
|
||||
txo.get_address(self.ledger), txo.channel.claim, name=txo.claim_name
|
||||
)
|
||||
except BadSignatureError:
|
||||
output['valid_signature'] = False
|
||||
|
||||
@@ -2,6 +2,7 @@ import sqlite3
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from binascii import hexlify
|
||||
from lbrynet import conf
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
|
||||
@@ -185,10 +186,14 @@ def do_migration(db_dir):
|
||||
(stream_hash, blob_hash, position, iv)
|
||||
)
|
||||
|
||||
download_dir = conf.settings.download_dir
|
||||
if not isinstance(download_dir, bytes):
|
||||
download_dir = download_dir.encode()
|
||||
|
||||
# insert the file
|
||||
transaction.execute(
|
||||
"insert or ignore into file values (?, ?, ?, ?, ?)",
|
||||
(stream_hash, stream_name, conf.settings.download_dir.encode('hex'),
|
||||
(stream_hash, stream_name, hexlify(download_dir),
|
||||
data_rate, status)
|
||||
)
|
||||
|
||||
|
||||
@@ -144,6 +144,7 @@ types_map = {
|
||||
'.pict': 'image/pict',
|
||||
'.rtf': 'application/rtf',
|
||||
'.xul': 'text/xul',
|
||||
'.m4v': 'video/m4v',
|
||||
|
||||
# microsoft is special and has its own 'standard'
|
||||
# https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
@@ -151,9 +152,7 @@ types_map = {
|
||||
}
|
||||
|
||||
|
||||
fallback_mime_type = 'application/octet-stream'
|
||||
|
||||
|
||||
def guess_mime_type(path):
|
||||
def guess_media_type(path):
|
||||
_, ext = os.path.splitext(path)
|
||||
return types_map.get(ext, fallback_mime_type)
|
||||
default = f'application/x-ext-{ext}' if ext else 'application/octet-stream'
|
||||
return types_map.get(ext, default)
|
||||
+146
-216
@@ -1,17 +1,16 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import traceback
|
||||
import typing
|
||||
from binascii import hexlify, unhexlify
|
||||
from twisted.internet import defer, task, threads
|
||||
from twisted.enterprise import adbapi
|
||||
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
|
||||
from lbrynet import conf
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.blob.CryptBlob import CryptBlobInfo
|
||||
from lbrynet.dht.constants import dataExpireTimeout
|
||||
from torba.client.basedatabase import SQLiteMixin
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -44,63 +43,23 @@ def _open_file_for_writing(download_directory, suggested_file_name):
|
||||
return os.path.basename(file_path)
|
||||
|
||||
|
||||
def open_file_for_writing(download_directory, suggested_file_name):
|
||||
"""
|
||||
Used to touch the path of a file to be downloaded
|
||||
|
||||
:param download_directory: (str)
|
||||
:param suggested_file_name: (str)
|
||||
:return: (str) basename
|
||||
"""
|
||||
return threads.deferToThread(_open_file_for_writing, download_directory, suggested_file_name)
|
||||
async def open_file_for_writing(download_directory: str, suggested_file_name: str) -> str:
|
||||
""" Used to touch the path of a file to be downloaded. """
|
||||
return await asyncio.get_event_loop().run_in_executor(
|
||||
None, _open_file_for_writing, download_directory, suggested_file_name
|
||||
)
|
||||
|
||||
|
||||
def rerun_if_locked(f):
|
||||
max_attempts = 5
|
||||
|
||||
def rerun(err, rerun_count, *args, **kwargs):
|
||||
connection = args[0]
|
||||
reactor = connection.reactor
|
||||
log.debug("Failed to execute (%s): %s", err, args)
|
||||
if err.check(sqlite3.OperationalError) and "database is locked" in str(err.value):
|
||||
log.warning("database was locked. rerunning %s with args %s, kwargs %s",
|
||||
str(f), str(args), str(kwargs))
|
||||
if rerun_count < max_attempts:
|
||||
delay = 2**rerun_count
|
||||
return task.deferLater(reactor, delay, inner_wrapper, rerun_count + 1, *args, **kwargs)
|
||||
raise err
|
||||
|
||||
def check_needed_rerun(result, rerun_count):
|
||||
if rerun_count:
|
||||
log.info("successfully reran database query")
|
||||
return result
|
||||
|
||||
def inner_wrapper(rerun_count, *args, **kwargs):
|
||||
d = f(*args, **kwargs)
|
||||
d.addCallback(check_needed_rerun, rerun_count)
|
||||
d.addErrback(rerun, rerun_count, *args, **kwargs)
|
||||
return d
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
return inner_wrapper(0, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
async def looping_call(interval, fun):
|
||||
while True:
|
||||
try:
|
||||
await fun()
|
||||
except Exception as e:
|
||||
log.exception('Looping call experienced exception:', exc_info=e)
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
|
||||
class SqliteConnection(adbapi.ConnectionPool):
|
||||
def __init__(self, db_path):
|
||||
super().__init__('sqlite3', db_path, check_same_thread=False)
|
||||
|
||||
@rerun_if_locked
|
||||
def runInteraction(self, interaction, *args, **kw):
|
||||
return super().runInteraction(interaction, *args, **kw)
|
||||
|
||||
@classmethod
|
||||
def set_reactor(cls, reactor):
|
||||
cls.reactor = reactor
|
||||
|
||||
|
||||
class SQLiteStorage:
|
||||
class SQLiteStorage(SQLiteMixin):
|
||||
|
||||
CREATE_TABLES_QUERY = """
|
||||
pragma foreign_keys=on;
|
||||
@@ -173,70 +132,46 @@ class SQLiteStorage:
|
||||
);
|
||||
"""
|
||||
|
||||
def __init__(self, db_dir, reactor=None):
|
||||
if not reactor:
|
||||
from twisted.internet import reactor
|
||||
self.db_dir = db_dir
|
||||
self._db_path = os.path.join(db_dir, "lbrynet.sqlite")
|
||||
log.info("connecting to database: %s", self._db_path)
|
||||
self.db = SqliteConnection(self._db_path)
|
||||
self.db.set_reactor(reactor)
|
||||
self.clock = reactor
|
||||
|
||||
# used to refresh the claim attributes on a ManagedEncryptedFileDownloader when a
|
||||
# change to the associated content claim occurs. these are added by the file manager
|
||||
# when it loads each file
|
||||
self.content_claim_callbacks = {} # {<stream_hash>: <callable returning a deferred>}
|
||||
def __init__(self, path, loop=None):
|
||||
super().__init__(path)
|
||||
self.content_claim_callbacks = {}
|
||||
self.check_should_announce_lc = None
|
||||
self.loop = loop or asyncio.get_event_loop()
|
||||
|
||||
async def open(self):
|
||||
await super().open()
|
||||
if 'reflector' not in conf.settings['components_to_skip']:
|
||||
self.check_should_announce_lc = task.LoopingCall(self.verify_will_announce_all_head_and_sd_blobs)
|
||||
self.check_should_announce_lc = looping_call(
|
||||
600, self.verify_will_announce_all_head_and_sd_blobs
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
def _create_tables(transaction):
|
||||
transaction.executescript(self.CREATE_TABLES_QUERY)
|
||||
yield self.db.runInteraction(_create_tables)
|
||||
if self.check_should_announce_lc and not self.check_should_announce_lc.running:
|
||||
self.check_should_announce_lc.start(600)
|
||||
defer.returnValue(None)
|
||||
async def close(self):
|
||||
if self.check_should_announce_lc is not None:
|
||||
self.check_should_announce_lc.close()
|
||||
await super().close()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run_and_return_one_or_none(self, query, *args):
|
||||
result = yield self.db.runQuery(query, args)
|
||||
if result:
|
||||
defer.returnValue(result[0][0])
|
||||
else:
|
||||
defer.returnValue(None)
|
||||
async def run_and_return_one_or_none(self, query, *args):
|
||||
for row in await self.db.execute_fetchall(query, args):
|
||||
if len(row) == 1:
|
||||
return row[0]
|
||||
return row
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def run_and_return_list(self, query, *args):
|
||||
result = yield self.db.runQuery(query, args)
|
||||
if result:
|
||||
defer.returnValue([i[0] for i in result])
|
||||
else:
|
||||
defer.returnValue([])
|
||||
async def run_and_return_list(self, query, *args):
|
||||
rows = list(await self.db.execute_fetchall(query, args))
|
||||
return [col[0] for col in rows] if rows else []
|
||||
|
||||
def run_and_return_id(self, query, *args):
|
||||
def do_save(t):
|
||||
t.execute(query, args)
|
||||
return t.lastrowid
|
||||
return self.db.runInteraction(do_save)
|
||||
|
||||
def stop(self):
|
||||
if self.check_should_announce_lc and self.check_should_announce_lc.running:
|
||||
self.check_should_announce_lc.stop()
|
||||
self.db.close()
|
||||
return defer.succeed(True)
|
||||
async def run_and_return_id(self, query, *args):
|
||||
return (await self.db.execute(query, args)).lastrowid
|
||||
|
||||
# # # # # # # # # blob functions # # # # # # # # #
|
||||
|
||||
def add_completed_blob(self, blob_hash, length, next_announce_time, should_announce, status="finished"):
|
||||
log.debug("Adding a completed blob. blob_hash=%s, length=%i", blob_hash, length)
|
||||
values = (blob_hash, length, next_announce_time or 0, int(bool(should_announce)), status, 0, 0)
|
||||
return self.db.runOperation("insert or replace into blob values (?, ?, ?, ?, ?, ?, ?)", values)
|
||||
return self.db.execute("insert or replace into blob values (?, ?, ?, ?, ?, ?, ?)", values)
|
||||
|
||||
def set_should_announce(self, blob_hash, next_announce_time, should_announce):
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"update blob set next_announce_time=?, should_announce=? where blob_hash=?",
|
||||
(next_announce_time or 0, int(bool(should_announce)), blob_hash)
|
||||
)
|
||||
@@ -246,9 +181,8 @@ class SQLiteStorage:
|
||||
"select status from blob where blob_hash=?", blob_hash
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_known_blob(self, blob_hash, length):
|
||||
yield self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"insert or ignore into blob values (?, ?, ?, ?, ?, ?, ?)", (blob_hash, length, 0, 0, "pending", 0, 0)
|
||||
)
|
||||
|
||||
@@ -267,12 +201,11 @@ class SQLiteStorage:
|
||||
"select blob_hash from blob where should_announce=1 and status='finished'"
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_all_finished_blobs(self):
|
||||
blob_hashes = yield self.run_and_return_list(
|
||||
async def get_all_finished_blobs(self):
|
||||
blob_hashes = await self.run_and_return_list(
|
||||
"select blob_hash from blob where status='finished'"
|
||||
)
|
||||
defer.returnValue([unhexlify(blob_hash) for blob_hash in blob_hashes])
|
||||
return [unhexlify(blob_hash) for blob_hash in blob_hashes]
|
||||
|
||||
def count_finished_blobs(self):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -280,14 +213,14 @@ class SQLiteStorage:
|
||||
)
|
||||
|
||||
def update_last_announced_blob(self, blob_hash, last_announced):
|
||||
return self.db.runOperation(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 where blob_hash=?",
|
||||
(int(last_announced + (dataExpireTimeout / 2)), int(last_announced), blob_hash)
|
||||
)
|
||||
return self.db.execute(
|
||||
"update blob set next_announce_time=?, last_announced_time=?, single_announce=0 where blob_hash=?",
|
||||
(int(last_announced + (dataExpireTimeout / 2)), int(last_announced), blob_hash)
|
||||
)
|
||||
|
||||
def should_single_announce_blobs(self, blob_hashes, immediate=False):
|
||||
def set_single_announce(transaction):
|
||||
now = self.clock.seconds()
|
||||
now = self.loop.time()
|
||||
for blob_hash in blob_hashes:
|
||||
if immediate:
|
||||
transaction.execute(
|
||||
@@ -298,11 +231,11 @@ class SQLiteStorage:
|
||||
transaction.execute(
|
||||
"update blob set single_announce=1 where blob_hash=? and status='finished'", (blob_hash, )
|
||||
)
|
||||
return self.db.runInteraction(set_single_announce)
|
||||
return self.db.run(set_single_announce)
|
||||
|
||||
def get_blobs_to_announce(self):
|
||||
def get_and_update(transaction):
|
||||
timestamp = self.clock.seconds()
|
||||
timestamp = self.loop.time()
|
||||
if conf.settings['announce_head_blobs_only']:
|
||||
r = transaction.execute(
|
||||
"select blob_hash from blob "
|
||||
@@ -317,13 +250,13 @@ class SQLiteStorage:
|
||||
)
|
||||
blobs = [b[0] for b in r.fetchall()]
|
||||
return blobs
|
||||
return self.db.runInteraction(get_and_update)
|
||||
return self.db.run(get_and_update)
|
||||
|
||||
def delete_blobs_from_db(self, blob_hashes):
|
||||
def delete_blobs(transaction):
|
||||
for blob_hash in blob_hashes:
|
||||
transaction.execute("delete from blob where blob_hash=?;", (blob_hash,))
|
||||
return self.db.runInteraction(delete_blobs)
|
||||
return self.db.run(delete_blobs)
|
||||
|
||||
def get_all_blob_hashes(self):
|
||||
return self.run_and_return_list("select blob_hash from blob")
|
||||
@@ -336,17 +269,16 @@ class SQLiteStorage:
|
||||
transaction.execute("insert into stream_blob values (?, ?, ?, ?)",
|
||||
(stream_hash, blob_info.get('blob_hash', None),
|
||||
blob_info['blob_num'], blob_info['iv']))
|
||||
return self.db.runInteraction(_add_stream_blobs)
|
||||
return self.db.run(_add_stream_blobs)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def add_known_blobs(self, blob_infos):
|
||||
async def add_known_blobs(self, blob_infos):
|
||||
for blob_info in blob_infos:
|
||||
if blob_info.get('blob_hash') and blob_info['length']:
|
||||
yield self.add_known_blob(blob_info['blob_hash'], blob_info['length'])
|
||||
await self.add_known_blob(blob_info['blob_hash'], blob_info['length'])
|
||||
|
||||
def verify_will_announce_head_and_sd_blobs(self, stream_hash):
|
||||
# fix should_announce for imported head and sd blobs
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"update blob set should_announce=1 "
|
||||
"where should_announce=0 and "
|
||||
"blob.blob_hash in "
|
||||
@@ -358,7 +290,7 @@ class SQLiteStorage:
|
||||
)
|
||||
|
||||
def verify_will_announce_all_head_and_sd_blobs(self):
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"update blob set should_announce=1 "
|
||||
"where should_announce=0 and "
|
||||
"blob.blob_hash in "
|
||||
@@ -383,23 +315,24 @@ class SQLiteStorage:
|
||||
:param stream_blob_infos: (list) of blob info dictionaries
|
||||
:return: (defer.Deferred)
|
||||
"""
|
||||
|
||||
def _store_stream(transaction):
|
||||
transaction.execute("insert into stream values (?, ?, ?, ?, ?);",
|
||||
(stream_hash, sd_hash, stream_key, stream_name,
|
||||
suggested_file_name))
|
||||
|
||||
transaction.execute(
|
||||
"insert into stream values (?, ?, ?, ?, ?);", (
|
||||
stream_hash, sd_hash, stream_key, stream_name, suggested_file_name
|
||||
)
|
||||
)
|
||||
for blob_info in stream_blob_infos:
|
||||
transaction.execute("insert into stream_blob values (?, ?, ?, ?)",
|
||||
(stream_hash, blob_info.get('blob_hash', None),
|
||||
blob_info['blob_num'], blob_info['iv']))
|
||||
transaction.execute(
|
||||
"insert into stream_blob values (?, ?, ?, ?)", (
|
||||
stream_hash, blob_info.get('blob_hash', None),
|
||||
blob_info['blob_num'], blob_info['iv']
|
||||
)
|
||||
)
|
||||
return self.db.run(_store_stream)
|
||||
|
||||
return self.db.runInteraction(_store_stream)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def delete_stream(self, stream_hash):
|
||||
sd_hash = yield self.get_sd_blob_hash_for_stream(stream_hash)
|
||||
stream_blobs = yield self.get_blobs_for_stream(stream_hash)
|
||||
async def delete_stream(self, stream_hash):
|
||||
sd_hash = await self.get_sd_blob_hash_for_stream(stream_hash)
|
||||
stream_blobs = await self.get_blobs_for_stream(stream_hash)
|
||||
blob_hashes = [b.blob_hash for b in stream_blobs if b.blob_hash is not None]
|
||||
|
||||
def _delete_stream(transaction):
|
||||
@@ -410,21 +343,25 @@ class SQLiteStorage:
|
||||
transaction.execute("delete from blob where blob_hash=?", (sd_hash, ))
|
||||
for blob_hash in blob_hashes:
|
||||
transaction.execute("delete from blob where blob_hash=?;", (blob_hash, ))
|
||||
yield self.db.runInteraction(_delete_stream)
|
||||
|
||||
await self.db.run(_delete_stream)
|
||||
|
||||
def get_all_streams(self):
|
||||
return self.run_and_return_list("select stream_hash from stream")
|
||||
|
||||
def get_stream_info(self, stream_hash):
|
||||
d = self.db.runQuery("select stream_name, stream_key, suggested_filename, sd_hash from stream "
|
||||
"where stream_hash=?", (stream_hash, ))
|
||||
d.addCallback(lambda r: None if not r else r[0])
|
||||
return d
|
||||
return self.run_and_return_one_or_none(
|
||||
"select stream_name, stream_key, suggested_filename, sd_hash from stream "
|
||||
"where stream_hash=?", stream_hash
|
||||
)
|
||||
|
||||
def check_if_stream_exists(self, stream_hash):
|
||||
d = self.db.runQuery("select stream_hash from stream where stream_hash=?", (stream_hash, ))
|
||||
d.addCallback(lambda r: bool(len(r)))
|
||||
return d
|
||||
async def check_if_stream_exists(self, stream_hash):
|
||||
row = await self.run_and_return_one_or_none(
|
||||
"select stream_hash from stream where stream_hash=?", stream_hash
|
||||
)
|
||||
if row is not None:
|
||||
return bool(len(row))
|
||||
return False
|
||||
|
||||
def get_blob_num_by_hash(self, stream_hash, blob_hash):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -466,7 +403,7 @@ class SQLiteStorage:
|
||||
crypt_blob_infos.append(CryptBlobInfo(blob_hash, position, blob_length, iv))
|
||||
crypt_blob_infos = sorted(crypt_blob_infos, key=lambda info: info.blob_num)
|
||||
return crypt_blob_infos
|
||||
return self.db.runInteraction(_get_blobs_for_stream)
|
||||
return self.db.run(_get_blobs_for_stream)
|
||||
|
||||
def get_pending_blobs_for_stream(self, stream_hash):
|
||||
return self.run_and_return_list(
|
||||
@@ -493,14 +430,12 @@ class SQLiteStorage:
|
||||
|
||||
# # # # # # # # # file stuff # # # # # # # # #
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
|
||||
async def save_downloaded_file(self, stream_hash, file_name, download_directory, data_payment_rate):
|
||||
# touch the closest available file to the file name
|
||||
file_name = yield open_file_for_writing(unhexlify(download_directory).decode(), unhexlify(file_name).decode())
|
||||
result = yield self.save_published_file(
|
||||
file_name = await open_file_for_writing(unhexlify(download_directory).decode(), unhexlify(file_name).decode())
|
||||
return await self.save_published_file(
|
||||
stream_hash, hexlify(file_name.encode()), download_directory, data_payment_rate
|
||||
)
|
||||
defer.returnValue(result)
|
||||
|
||||
def save_published_file(self, stream_hash, file_name, download_directory, data_payment_rate, status="stopped"):
|
||||
return self.run_and_return_id(
|
||||
@@ -509,7 +444,9 @@ class SQLiteStorage:
|
||||
)
|
||||
|
||||
def get_filename_for_rowid(self, rowid):
|
||||
return self.run_and_return_one_or_none("select file_name from file where rowid=?", rowid)
|
||||
return self.run_and_return_one_or_none(
|
||||
"select file_name from file where rowid=?", rowid
|
||||
)
|
||||
|
||||
def get_all_lbry_files(self):
|
||||
def _lbry_file_dict(rowid, stream_hash, file_name, download_dir, data_rate, status, _, sd_hash, stream_key,
|
||||
@@ -535,13 +472,11 @@ class SQLiteStorage:
|
||||
).fetchall()
|
||||
]
|
||||
|
||||
d = self.db.runInteraction(_get_all_files)
|
||||
return d
|
||||
return self.db.run(_get_all_files)
|
||||
|
||||
def change_file_status(self, rowid, new_status):
|
||||
d = self.db.runQuery("update file set status=? where rowid=?", (new_status, rowid))
|
||||
d.addCallback(lambda _: new_status)
|
||||
return d
|
||||
async def change_file_status(self, rowid, new_status):
|
||||
await self.db.execute("update file set status=? where rowid=?", (new_status, rowid))
|
||||
return new_status
|
||||
|
||||
def get_lbry_file_status(self, rowid):
|
||||
return self.run_and_return_one_or_none(
|
||||
@@ -565,7 +500,7 @@ class SQLiteStorage:
|
||||
("%s:%i" % (support['txid'], support['nout']), claim_id, lbc_to_dewies(support['amount']),
|
||||
support.get('address', ""))
|
||||
)
|
||||
return self.db.runInteraction(_save_support)
|
||||
return self.db.run(_save_support)
|
||||
|
||||
def get_supports(self, *claim_ids):
|
||||
def _format_support(outpoint, supported_id, amount, address):
|
||||
@@ -587,15 +522,16 @@ class SQLiteStorage:
|
||||
)
|
||||
]
|
||||
|
||||
return self.db.runInteraction(_get_supports)
|
||||
return self.db.run(_get_supports)
|
||||
|
||||
# # # # # # # # # claim functions # # # # # # # # #
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def save_claims(self, claim_infos):
|
||||
async def save_claims(self, claim_infos):
|
||||
support_callbacks = []
|
||||
update_file_callbacks = []
|
||||
|
||||
def _save_claims(transaction):
|
||||
content_claims_to_update = []
|
||||
support_callbacks = []
|
||||
for claim_info in claim_infos:
|
||||
outpoint = "%s:%i" % (claim_info['txid'], claim_info['nout'])
|
||||
claim_id = claim_info['claim_id']
|
||||
@@ -605,7 +541,7 @@ class SQLiteStorage:
|
||||
address = claim_info['address']
|
||||
sequence = claim_info['claim_sequence']
|
||||
try:
|
||||
certificate_id = claim_info['value'].get('content_claims_to_update', {}).get('certificateId')
|
||||
certificate_id = claim_info['value'].get('publisherSignature', {}).get('certificateId')
|
||||
except AttributeError:
|
||||
certificate_id = None
|
||||
try:
|
||||
@@ -622,7 +558,7 @@ class SQLiteStorage:
|
||||
)
|
||||
if 'supports' in claim_info: # if this response doesn't have support info don't overwrite the existing
|
||||
# support info
|
||||
support_callbacks.append(self.save_supports(claim_id, claim_info['supports']))
|
||||
support_callbacks.append((claim_id, claim_info['supports']))
|
||||
if not source_hash:
|
||||
continue
|
||||
stream_hash = transaction.execute(
|
||||
@@ -644,18 +580,18 @@ class SQLiteStorage:
|
||||
content_claims_to_update.append((stream_hash, outpoint))
|
||||
elif known_outpoint != outpoint:
|
||||
content_claims_to_update.append((stream_hash, outpoint))
|
||||
update_file_callbacks = []
|
||||
for stream_hash, outpoint in content_claims_to_update:
|
||||
self._save_content_claim(transaction, outpoint, stream_hash)
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
update_file_callbacks.append(self.content_claim_callbacks[stream_hash]())
|
||||
return update_file_callbacks, support_callbacks
|
||||
|
||||
content_dl, support_dl = yield self.db.runInteraction(_save_claims)
|
||||
if content_dl:
|
||||
yield defer.DeferredList(content_dl)
|
||||
if support_dl:
|
||||
yield defer.DeferredList(support_dl)
|
||||
await self.db.run(_save_claims)
|
||||
if update_file_callbacks:
|
||||
await asyncio.wait(update_file_callbacks)
|
||||
if support_callbacks:
|
||||
await asyncio.wait([
|
||||
self.save_supports(*args) for args in support_callbacks
|
||||
])
|
||||
|
||||
def save_claims_for_resolve(self, claim_infos):
|
||||
to_save = []
|
||||
@@ -718,16 +654,13 @@ class SQLiteStorage:
|
||||
# update the claim associated to the file
|
||||
transaction.execute("insert or replace into content_claim values (?, ?)", (stream_hash, claim_outpoint))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def save_content_claim(self, stream_hash, claim_outpoint):
|
||||
yield self.db.runInteraction(self._save_content_claim, claim_outpoint, stream_hash)
|
||||
async def save_content_claim(self, stream_hash, claim_outpoint):
|
||||
await self.db.run(self._save_content_claim, claim_outpoint, stream_hash)
|
||||
# update corresponding ManagedEncryptedFileDownloader object
|
||||
if stream_hash in self.content_claim_callbacks:
|
||||
file_callback = self.content_claim_callbacks[stream_hash]
|
||||
yield file_callback()
|
||||
await self.content_claim_callbacks[stream_hash]()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_content_claim(self, stream_hash, include_supports=True):
|
||||
async def get_content_claim(self, stream_hash, include_supports=True):
|
||||
def _get_claim_from_stream_hash(transaction):
|
||||
claim_info = transaction.execute(
|
||||
"select c.*, "
|
||||
@@ -745,15 +678,13 @@ class SQLiteStorage:
|
||||
result['channel_name'] = channel_name
|
||||
return result
|
||||
|
||||
result = yield self.db.runInteraction(_get_claim_from_stream_hash)
|
||||
result = await self.db.run(_get_claim_from_stream_hash)
|
||||
if result and include_supports:
|
||||
supports = yield self.get_supports(result['claim_id'])
|
||||
result['supports'] = supports
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], supports)
|
||||
defer.returnValue(result)
|
||||
result['supports'] = await self.get_supports(result['claim_id'])
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], result['supports'])
|
||||
return result
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claims_from_stream_hashes(self, stream_hashes, include_supports=True):
|
||||
async def get_claims_from_stream_hashes(self, stream_hashes, include_supports=True):
|
||||
def _batch_get_claim(transaction):
|
||||
results = {}
|
||||
claim_infos = _batched_select(
|
||||
@@ -781,10 +712,10 @@ class SQLiteStorage:
|
||||
results[stream_hash]['channel_name'] = channel_name
|
||||
return results
|
||||
|
||||
claims = yield self.db.runInteraction(_batch_get_claim)
|
||||
claims = await self.db.run(_batch_get_claim)
|
||||
if include_supports:
|
||||
all_supports = {}
|
||||
for support in (yield self.get_supports(*[claim['claim_id'] for claim in claims.values()])):
|
||||
for support in await self.get_supports(*[claim['claim_id'] for claim in claims.values()]):
|
||||
all_supports.setdefault(support['claim_id'], []).append(support)
|
||||
for stream_hash in claims.keys():
|
||||
claim = claims[stream_hash]
|
||||
@@ -792,28 +723,28 @@ class SQLiteStorage:
|
||||
claim['supports'] = supports
|
||||
claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports)
|
||||
claims[stream_hash] = claim
|
||||
defer.returnValue(claims)
|
||||
return claims
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_claim(self, claim_outpoint, include_supports=True):
|
||||
async def get_claim(self, claim_outpoint, include_supports=True):
|
||||
def _get_claim(transaction):
|
||||
claim_info = transaction.execute("select c.*, "
|
||||
"case when c.channel_claim_id is not null then "
|
||||
"(select claim_name from claim where claim_id==c.channel_claim_id) "
|
||||
"else null end as channel_name from claim c where claim_outpoint = ?",
|
||||
(claim_outpoint,)).fetchone()
|
||||
claim_info = transaction.execute(
|
||||
"select c.*, "
|
||||
"case when c.channel_claim_id is not null then "
|
||||
"(select claim_name from claim where claim_id==c.channel_claim_id) "
|
||||
"else null end as channel_name from claim c where claim_outpoint = ?",
|
||||
(claim_outpoint,)
|
||||
).fetchone()
|
||||
channel_name = claim_info[-1]
|
||||
result = _format_claim_response(*claim_info[:-1])
|
||||
if channel_name:
|
||||
result['channel_name'] = channel_name
|
||||
return result
|
||||
|
||||
result = yield self.db.runInteraction(_get_claim)
|
||||
result = await self.db.run(_get_claim)
|
||||
if include_supports:
|
||||
supports = yield self.get_supports(result['claim_id'])
|
||||
result['supports'] = supports
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], supports)
|
||||
defer.returnValue(result)
|
||||
result['supports'] = await self.get_supports(result['claim_id'])
|
||||
result['effective_amount'] = calculate_effective_amount(result['amount'], result['supports'])
|
||||
return result
|
||||
|
||||
def get_unknown_certificate_ids(self):
|
||||
def _get_unknown_certificate_claim_ids(transaction):
|
||||
@@ -825,11 +756,10 @@ class SQLiteStorage:
|
||||
"(select c2.claim_id from claim as c2)"
|
||||
).fetchall()
|
||||
]
|
||||
return self.db.runInteraction(_get_unknown_certificate_claim_ids)
|
||||
return self.db.run(_get_unknown_certificate_claim_ids)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_pending_claim_outpoints(self):
|
||||
claim_outpoints = yield self.run_and_return_list("select claim_outpoint from claim where height=-1")
|
||||
async def get_pending_claim_outpoints(self):
|
||||
claim_outpoints = await self.run_and_return_list("select claim_outpoint from claim where height=-1")
|
||||
results = {} # {txid: [nout, ...]}
|
||||
for outpoint_str in claim_outpoints:
|
||||
txid, nout = outpoint_str.split(":")
|
||||
@@ -838,7 +768,7 @@ class SQLiteStorage:
|
||||
results[txid] = outputs
|
||||
if results:
|
||||
log.debug("missing transaction heights for %i claims", len(results))
|
||||
defer.returnValue(results)
|
||||
return results
|
||||
|
||||
def save_claim_tx_heights(self, claim_tx_heights):
|
||||
def _save_claim_heights(transaction):
|
||||
@@ -847,17 +777,17 @@ class SQLiteStorage:
|
||||
"update claim set height=? where claim_outpoint=? and height=-1",
|
||||
(height, outpoint)
|
||||
)
|
||||
return self.db.runInteraction(_save_claim_heights)
|
||||
return self.db.run(_save_claim_heights)
|
||||
|
||||
# # # # # # # # # reflector functions # # # # # # # # #
|
||||
|
||||
def update_reflected_stream(self, sd_hash, reflector_address, success=True):
|
||||
if success:
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"insert or replace into reflected_stream values (?, ?, ?)",
|
||||
(sd_hash, reflector_address, self.clock.seconds())
|
||||
(sd_hash, reflector_address, self.loop.time())
|
||||
)
|
||||
return self.db.runOperation(
|
||||
return self.db.execute(
|
||||
"delete from reflected_stream where sd_hash=? and reflector_address=?",
|
||||
(sd_hash, reflector_address)
|
||||
)
|
||||
@@ -867,7 +797,7 @@ class SQLiteStorage:
|
||||
"select s.sd_hash from stream s "
|
||||
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
|
||||
"where r.timestamp is null or r.timestamp < ?",
|
||||
self.clock.seconds() - conf.settings['auto_re_reflect_interval']
|
||||
self.loop.time() - conf.settings['auto_re_reflect_interval']
|
||||
)
|
||||
|
||||
|
||||
|
||||
File renamed without changes.
@@ -165,15 +165,10 @@ class BlobReflectorClient(Protocol):
|
||||
blob_hash = self.blob_hashes_to_send[0]
|
||||
log.debug('No current blob, sending the next one: %s', blob_hash)
|
||||
self.blob_hashes_to_send = self.blob_hashes_to_send[1:]
|
||||
d = self.blob_manager.get_blob(blob_hash)
|
||||
d.addCallback(self.open_blob_for_reading)
|
||||
blob = self.blob_manager.get_blob(blob_hash)
|
||||
self.open_blob_for_reading(blob)
|
||||
# send the server the next blob hash + length
|
||||
d.addCallbacks(
|
||||
lambda _: self.send_blob_info(),
|
||||
errback=log.fail(self.disconnect),
|
||||
errbackArgs=("Error reflecting blob %s", blob_hash)
|
||||
)
|
||||
return d
|
||||
return self.send_blob_info()
|
||||
else:
|
||||
# close connection
|
||||
log.debug('No more blob hashes, closing connection')
|
||||
|
||||
@@ -6,6 +6,7 @@ from twisted.protocols.basic import FileSender
|
||||
from twisted.internet.protocol import Protocol, ClientFactory
|
||||
from twisted.internet import defer, error
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.reflector.common import IncompleteResponse, ReflectorRequestError
|
||||
from lbrynet.extras.reflector.common import REFLECTOR_V1, REFLECTOR_V2
|
||||
|
||||
@@ -61,7 +62,9 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
else:
|
||||
reflected = False
|
||||
|
||||
d = self.blob_manager.storage.update_reflected_stream(self.sd_hash, self.transport.getPeer().host, reflected)
|
||||
d = f2d(self.blob_manager.storage.update_reflected_stream(
|
||||
self.sd_hash, self.transport.getPeer().host, reflected
|
||||
))
|
||||
d.addCallback(lambda _: result)
|
||||
return d
|
||||
|
||||
@@ -113,10 +116,7 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
for crypt_blob in blobs:
|
||||
if crypt_blob.blob_hash and crypt_blob.length:
|
||||
yield self.blob_manager.get_blob(crypt_blob.blob_hash, crypt_blob.length)
|
||||
|
||||
dl = defer.DeferredList(list(get_blobs(blobs_in_stream)), consumeErrors=True)
|
||||
dl.addCallback(lambda blobs: [blob for r, blob in blobs if r and blob.get_is_verified()])
|
||||
return dl
|
||||
return [blob for blob in get_blobs(blobs_in_stream) if blob.get_is_verified()]
|
||||
|
||||
def set_blobs_to_send(self, blobs_to_send):
|
||||
for blob in blobs_to_send:
|
||||
@@ -132,7 +132,7 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
len(filtered))
|
||||
return filtered
|
||||
|
||||
d = self.factory.blob_manager.storage.get_blobs_for_stream(self.stream_hash)
|
||||
d = f2d(self.factory.blob_manager.storage.get_blobs_for_stream(self.stream_hash))
|
||||
d.addCallback(self.get_validated_blobs)
|
||||
if not self.descriptor_needed:
|
||||
d.addCallback(lambda filtered:
|
||||
@@ -303,7 +303,7 @@ class EncryptedFileReflectorClient(Protocol):
|
||||
# open the next blob to send
|
||||
blob_hash = self.blob_hashes_to_send[0]
|
||||
self.blob_hashes_to_send = self.blob_hashes_to_send[1:]
|
||||
d = self.blob_manager.get_blob(blob_hash)
|
||||
d = defer.succeed(self.blob_manager.get_blob(blob_hash))
|
||||
d.addCallback(self.open_blob_for_reading)
|
||||
d.addCallbacks(lambda _: self.send_blob_info(),
|
||||
lambda err: self.skip_missing_blob(err, blob.blob_hash))
|
||||
|
||||
@@ -45,7 +45,7 @@ def _reflect_blobs(blob_manager, blob_hashes, reflector_server):
|
||||
ip = yield resolve(reflector_address)
|
||||
yield reactor.connectTCP(ip, reflector_port, factory)
|
||||
result = yield factory.finished_deferred
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
|
||||
|
||||
def reflect_file(lbry_file, reflector_server=None):
|
||||
|
||||
@@ -236,3 +236,6 @@ class Account(BaseAccount):
|
||||
)
|
||||
|
||||
return tx
|
||||
|
||||
async def release_all_outputs(self):
|
||||
await self.ledger.db.release_all_outputs(self)
|
||||
@@ -117,3 +117,11 @@ class WalletDatabase(BaseDatabase):
|
||||
channel.private_key = private_key
|
||||
certificates.append(channel)
|
||||
return certificates
|
||||
|
||||
async def release_all_outputs(self, account):
|
||||
await self.db.execute(
|
||||
"UPDATE txo SET is_reserved = 0 WHERE"
|
||||
" is_reserved = 1 AND txo.address IN ("
|
||||
" SELECT address from pubkey_address WHERE account = ?"
|
||||
" )", [account.public_key.address]
|
||||
)
|
||||
@@ -1,6 +1,5 @@
|
||||
import os
|
||||
import json
|
||||
import asyncio
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
|
||||
@@ -15,6 +14,7 @@ from torba.rpc.jsonrpc import CodeMessageError
|
||||
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.wallet.ledger import MainNetLedger
|
||||
from lbrynet.extras.wallet.account import BaseAccount, generate_certificate
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
@@ -277,7 +277,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
if 'error' not in results:
|
||||
await self.old_db.save_claims_for_resolve([
|
||||
value for value in results.values() if 'error' not in value
|
||||
]).asFuture(asyncio.get_event_loop())
|
||||
])
|
||||
return results
|
||||
|
||||
async def get_claims_for_name(self, name: str):
|
||||
@@ -413,7 +413,8 @@ class LbryWalletManager(BaseWalletManager):
|
||||
claim_address = await account.receiving.get_or_create_usable_address()
|
||||
if certificate:
|
||||
claim = claim.sign(
|
||||
certificate.private_key, claim_address, certificate.claim_id, curve=SECP256k1
|
||||
certificate.private_key, claim_address, certificate.claim_id, curve=SECP256k1, name=name,
|
||||
force_detached=False # TODO: delete it and make True default everywhere when its out
|
||||
)
|
||||
existing_claims = await account.get_claims(
|
||||
claim_name_type__any={'is_claim': 1, 'is_update': 1}, # exclude is_supports
|
||||
@@ -432,7 +433,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
await account.ledger.broadcast(tx)
|
||||
await self.old_db.save_claims([self._old_get_temp_claim_info(
|
||||
tx, tx.outputs[0], claim_address, claim_dict, name, dewies_to_lbc(amount)
|
||||
)]).asFuture(asyncio.get_event_loop())
|
||||
)])
|
||||
# TODO: release reserved tx outputs in case anything fails by this point
|
||||
return tx
|
||||
|
||||
@@ -446,7 +447,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
'address': holding_address,
|
||||
'claim_id': claim_id,
|
||||
'amount': dewies_to_lbc(amount)
|
||||
}]).asFuture(asyncio.get_event_loop())
|
||||
}])
|
||||
return tx
|
||||
|
||||
async def tip_claim(self, amount, claim_id, account):
|
||||
@@ -461,7 +462,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
'address': claim_to_tip['address'],
|
||||
'claim_id': claim_id,
|
||||
'amount': dewies_to_lbc(amount)
|
||||
}]).asFuture(asyncio.get_event_loop())
|
||||
}])
|
||||
return tx
|
||||
|
||||
async def abandon_claim(self, claim_id, txid, nout, account):
|
||||
@@ -484,7 +485,7 @@ class LbryWalletManager(BaseWalletManager):
|
||||
|
||||
await self.old_db.save_claims([self._old_get_temp_claim_info(
|
||||
tx, tx.outputs[0], address, cert, channel_name, dewies_to_lbc(amount)
|
||||
)]).asFuture(asyncio.get_event_loop())
|
||||
)])
|
||||
return tx
|
||||
|
||||
def _old_get_temp_claim_info(self, tx, txo, address, claim_dict, name, bid):
|
||||
@@ -611,8 +612,7 @@ class LBRYcrdAddressQueryHandler:
|
||||
@defer.inlineCallbacks
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
future = self.wallet.get_unused_address_for_peer(self.peer)
|
||||
address = yield defer.Deferred.fromFuture(asyncio.ensure_future(future))
|
||||
address = yield f2d(self.wallet.get_unused_address_for_peer(self.peer))
|
||||
self.address = address
|
||||
fields = {'lbrycrd_address': address}
|
||||
return fields
|
||||
|
||||
@@ -155,8 +155,8 @@ class Resolver:
|
||||
certificate = None
|
||||
certificate_id = smart_decode(claim_result['value']).certificate_id
|
||||
if certificate_id:
|
||||
certificate = await self.network.get_claims_by_ids(certificate_id.decode())
|
||||
certificate = certificate.pop(certificate_id.decode()) if certificate else None
|
||||
certificate = await self.network.get_claims_by_ids(certificate_id)
|
||||
certificate = certificate.pop(certificate_id) if certificate else None
|
||||
return await self.parse_and_validate_claim_result(claim_result, certificate=certificate)
|
||||
|
||||
async def parse_and_validate_claim_result(self, claim_result, certificate=None, raw=False):
|
||||
@@ -180,13 +180,13 @@ class Resolver:
|
||||
if decoded.has_signature:
|
||||
if certificate is None:
|
||||
log.info("fetching certificate to check claim signature")
|
||||
certificate = await self.network.get_claims_by_ids(decoded.certificate_id.decode())
|
||||
certificate = await self.network.get_claims_by_ids(decoded.certificate_id)
|
||||
if not certificate:
|
||||
log.warning('Certificate %s not found', decoded.certificate_id)
|
||||
claim_result['has_signature'] = True
|
||||
claim_result['signature_is_valid'] = False
|
||||
validated, channel_name = validate_claim_signature_and_get_channel_name(
|
||||
decoded, certificate, claim_result['address'])
|
||||
decoded, certificate, claim_result['address'], claim_result['name'])
|
||||
claim_result['channel_name'] = channel_name
|
||||
if validated:
|
||||
claim_result['signature_is_valid'] = True
|
||||
@@ -197,7 +197,7 @@ class Resolver:
|
||||
if 'amount' in claim_result:
|
||||
claim_result = format_amount_value(claim_result)
|
||||
|
||||
claim_result['permanent_url'] = _get_permanent_url(claim_result)
|
||||
claim_result['permanent_url'] = _get_permanent_url(claim_result, decoded.certificate_id)
|
||||
|
||||
return claim_result
|
||||
|
||||
@@ -307,18 +307,11 @@ def format_amount_value(obj):
|
||||
return obj
|
||||
|
||||
|
||||
def _get_permanent_url(claim_result):
|
||||
if claim_result.get('has_signature') and claim_result.get('channel_name'):
|
||||
return "{}#{}/{}".format(
|
||||
claim_result['channel_name'],
|
||||
claim_result['value']['publisherSignature']['certificateId'],
|
||||
claim_result['name']
|
||||
)
|
||||
def _get_permanent_url(claim_result, certificate_id):
|
||||
if certificate_id:
|
||||
return f"{claim_result['channel_name']}#{certificate_id}/{claim_result['name']}"
|
||||
else:
|
||||
return "{}#{}".format(
|
||||
claim_result['name'],
|
||||
claim_result['claim_id']
|
||||
)
|
||||
return f"{claim_result['name']}#{claim_result['claim_id']}"
|
||||
|
||||
|
||||
def _verify_proof(name, claim_trie_root, result, height, depth, transaction_class, hash160_to_address):
|
||||
@@ -386,7 +379,7 @@ def _verify_proof(name, claim_trie_root, result, height, depth, transaction_clas
|
||||
|
||||
|
||||
def validate_claim_signature_and_get_channel_name(claim, certificate_claim,
|
||||
claim_address, decoded_certificate=None):
|
||||
claim_address, name, decoded_certificate=None):
|
||||
if not certificate_claim:
|
||||
return False, None
|
||||
if 'value' not in certificate_claim:
|
||||
@@ -395,18 +388,18 @@ def validate_claim_signature_and_get_channel_name(claim, certificate_claim,
|
||||
certificate = decoded_certificate or smart_decode(certificate_claim['value'])
|
||||
if not isinstance(certificate, ClaimDict):
|
||||
raise TypeError("Certificate is not a ClaimDict: %s" % str(type(certificate)))
|
||||
if _validate_signed_claim(claim, claim_address, certificate):
|
||||
if _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
return True, certificate_claim['name']
|
||||
return False, None
|
||||
|
||||
|
||||
def _validate_signed_claim(claim, claim_address, certificate):
|
||||
def _validate_signed_claim(claim, claim_address, name, certificate):
|
||||
if not claim.has_signature:
|
||||
raise Exception("Claim is not signed")
|
||||
if not is_address(claim_address):
|
||||
raise Exception("Not given a valid claim address")
|
||||
try:
|
||||
if claim.validate_signature(claim_address, certificate.protobuf):
|
||||
if claim.validate_signature(claim_address, certificate.protobuf, name):
|
||||
return True
|
||||
except BadSignatureError:
|
||||
# print_msg("Signature for %s is invalid" % claim_id)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import hashlib
|
||||
import struct
|
||||
from binascii import unhexlify
|
||||
|
||||
import msgpack
|
||||
from torba.server.hash import hash_to_hex_str
|
||||
|
||||
from torba.server.block_processor import BlockProcessor
|
||||
from lbrynet.schema.proto.claim_pb2 import Claim
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
|
||||
@@ -131,9 +131,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
super().backup_blocks(raw_blocks=raw_blocks)
|
||||
self.db.batched_flush_claims()
|
||||
|
||||
def shutdown(self):
|
||||
self.db.shutdown()
|
||||
|
||||
async def flush(self, flush_utxos):
|
||||
self.db.batched_flush_claims()
|
||||
return await super().flush(flush_utxos)
|
||||
@@ -153,14 +150,14 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
def _checksig(self, name, value, address):
|
||||
try:
|
||||
parse_lbry_uri(name.decode()) # skip invalid names
|
||||
cert_id = Claim.FromString(value).publisherSignature.certificateId[::-1] or None
|
||||
claim_dict = smart_decode(value)
|
||||
cert_id = unhexlify(claim_dict.certificate_id)[::-1]
|
||||
if not self.should_validate_signatures:
|
||||
return cert_id
|
||||
if cert_id:
|
||||
cert_claim = self.db.get_claim_info(cert_id)
|
||||
if cert_claim:
|
||||
certificate = smart_decode(cert_claim.value)
|
||||
claim_dict = smart_decode(value)
|
||||
claim_dict.validate_signature(address, certificate)
|
||||
return cert_id
|
||||
except Exception as e:
|
||||
|
||||
@@ -137,3 +137,8 @@ class LBCRegTest(LBC):
|
||||
XPRV_VERBYTES = bytes.fromhex('04358394')
|
||||
P2PKH_VERBYTE = bytes.fromhex("6f")
|
||||
P2SH_VERBYTES = bytes.fromhex("c4")
|
||||
|
||||
|
||||
class LBCTestNet(LBCRegTest):
|
||||
NET = "testnet"
|
||||
GENESIS_HASH = '9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463'
|
||||
@@ -37,7 +37,7 @@ class LBCDaemon(Daemon):
|
||||
@handles_errors
|
||||
async def getclaimsfortx(self, txid):
|
||||
'''Given a txid, returns the claims it make.'''
|
||||
return await self._send_single('getclaimsfortx', (txid,))
|
||||
return await self._send_single('getclaimsfortx', (txid,)) or []
|
||||
|
||||
@handles_errors
|
||||
async def getnameproof(self, name, block_hash=None):
|
||||
|
||||
@@ -21,7 +21,7 @@ class LBRYDB(DB):
|
||||
self.pending_abandons = {}
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def shutdown(self):
|
||||
def close(self):
|
||||
self.batched_flush_claims()
|
||||
self.claims_db.close()
|
||||
self.names_db.close()
|
||||
@@ -29,9 +29,7 @@ class LBRYDB(DB):
|
||||
self.outpoint_to_claim_id_db.close()
|
||||
self.claim_undo_db.close()
|
||||
self.utxo_db.close()
|
||||
# electrumx ones
|
||||
self.utxo_db.close()
|
||||
self.history.close_db()
|
||||
super().close()
|
||||
|
||||
async def _open_dbs(self, for_sync, compacting):
|
||||
await super()._open_dbs(for_sync=for_sync, compacting=compacting)
|
||||
|
||||
@@ -307,7 +307,7 @@ class LBRYElectrumX(ElectrumX):
|
||||
return {'error': err.message}
|
||||
result = {}
|
||||
|
||||
if parsed_uri.is_channel:
|
||||
if parsed_uri.contains_channel:
|
||||
certificate = None
|
||||
|
||||
# TODO: this is also done on the else, refactor
|
||||
|
||||
+31
-36
@@ -1,8 +1,10 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from binascii import unhexlify
|
||||
from sqlite3 import IntegrityError
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.internet import defer
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.blob.creator import BlobFileCreator
|
||||
|
||||
@@ -26,15 +28,13 @@ class DiskBlobManager:
|
||||
self.blobs = {}
|
||||
self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)}
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setup(self):
|
||||
async def setup(self):
|
||||
if self._node_datastore is not None:
|
||||
raw_blob_hashes = yield self.storage.get_all_finished_blobs()
|
||||
raw_blob_hashes = await self.storage.get_all_finished_blobs()
|
||||
self._node_datastore.completed_blobs.update(raw_blob_hashes)
|
||||
defer.returnValue(True)
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(True)
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
def get_blob(self, blob_hash, length=None):
|
||||
"""Return a blob identified by blob_hash, which may be a new blob or a
|
||||
@@ -43,7 +43,7 @@ class DiskBlobManager:
|
||||
if length is not None and not isinstance(length, int):
|
||||
raise Exception("invalid length type: {} ({})".format(length, str(type(length))))
|
||||
if blob_hash in self.blobs:
|
||||
return defer.succeed(self.blobs[blob_hash])
|
||||
return self.blobs[blob_hash]
|
||||
return self._make_new_blob(blob_hash, length)
|
||||
|
||||
def get_blob_creator(self):
|
||||
@@ -53,13 +53,13 @@ class DiskBlobManager:
|
||||
log.debug('Making a new blob for %s', blob_hash)
|
||||
blob = BlobFile(self.blob_dir, blob_hash, length)
|
||||
self.blobs[blob_hash] = blob
|
||||
return defer.succeed(blob)
|
||||
return blob
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def blob_completed(self, blob, should_announce=False, next_announce_time=None):
|
||||
yield self.storage.add_completed_blob(
|
||||
yield f2d(self.storage.add_completed_blob(
|
||||
blob.blob_hash, blob.length, next_announce_time, should_announce
|
||||
)
|
||||
))
|
||||
if self._node_datastore is not None:
|
||||
self._node_datastore.completed_blobs.add(unhexlify(blob.blob_hash))
|
||||
|
||||
@@ -67,14 +67,15 @@ class DiskBlobManager:
|
||||
return self._completed_blobs(blobhashes_to_check)
|
||||
|
||||
def count_should_announce_blobs(self):
|
||||
return self.storage.count_should_announce_blobs()
|
||||
return f2d(self.storage.count_should_announce_blobs())
|
||||
|
||||
def set_should_announce(self, blob_hash, should_announce):
|
||||
now = self.storage.clock.seconds()
|
||||
return self.storage.set_should_announce(blob_hash, now, should_announce)
|
||||
return f2d(self.storage.set_should_announce(
|
||||
blob_hash, asyncio.get_event_loop().time(), should_announce
|
||||
))
|
||||
|
||||
def get_should_announce(self, blob_hash):
|
||||
return self.storage.should_announce(blob_hash)
|
||||
return f2d(self.storage.should_announce(blob_hash))
|
||||
|
||||
def creator_finished(self, blob_creator, should_announce):
|
||||
log.debug("blob_creator.blob_hash: %s", blob_creator.blob_hash)
|
||||
@@ -89,7 +90,7 @@ class DiskBlobManager:
|
||||
return self.blob_completed(new_blob, should_announce)
|
||||
|
||||
def get_all_verified_blobs(self):
|
||||
d = self._get_all_verified_blob_hashes()
|
||||
d = f2d(self._get_all_verified_blob_hashes())
|
||||
d.addCallback(self.completed_blobs)
|
||||
return d
|
||||
|
||||
@@ -105,35 +106,29 @@ class DiskBlobManager:
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
blob = yield self.get_blob(blob_hash)
|
||||
yield blob.delete()
|
||||
blob = self.get_blob(blob_hash)
|
||||
blob.delete()
|
||||
bh_to_delete_from_db.append(blob_hash)
|
||||
del self.blobs[blob_hash]
|
||||
except Exception as e:
|
||||
log.warning("Failed to delete blob file. Reason: %s", e)
|
||||
try:
|
||||
yield self.storage.delete_blobs_from_db(bh_to_delete_from_db)
|
||||
yield f2d(self.storage.delete_blobs_from_db(bh_to_delete_from_db))
|
||||
except IntegrityError as err:
|
||||
if str(err) != "FOREIGN KEY constraint failed":
|
||||
raise err
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _completed_blobs(self, blobhashes_to_check):
|
||||
"""Returns of the blobhashes_to_check, which are valid"""
|
||||
blobs = yield defer.DeferredList([self.get_blob(b) for b in blobhashes_to_check])
|
||||
blob_hashes = [b.blob_hash for success, b in blobs if success and b.verified]
|
||||
defer.returnValue(blob_hashes)
|
||||
blobs = [self.get_blob(b) for b in blobhashes_to_check]
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.verified]
|
||||
return blob_hashes
|
||||
|
||||
def _get_all_verified_blob_hashes(self):
|
||||
d = self.storage.get_all_blob_hashes()
|
||||
|
||||
def get_verified_blobs(blobs):
|
||||
verified_blobs = []
|
||||
for blob_hash in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
|
||||
d.addCallback(lambda blobs: threads.deferToThread(get_verified_blobs, blobs))
|
||||
return d
|
||||
async def _get_all_verified_blob_hashes(self):
|
||||
blobs = await self.storage.get_all_blob_hashes()
|
||||
verified_blobs = []
|
||||
for blob_hash in blobs:
|
||||
file_path = os.path.join(self.blob_dir, blob_hash)
|
||||
if os.path.isfile(file_path):
|
||||
verified_blobs.append(blob_hash)
|
||||
return verified_blobs
|
||||
@@ -5,7 +5,7 @@ from twisted.internet import defer, task
|
||||
from twisted.internet.error import ConnectingCancelledError
|
||||
from twisted.web._newclient import ResponseNeverReceived
|
||||
|
||||
from lbrynet.utils import DeferredDict
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.p2p.Error import DownloadCanceledError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -104,9 +104,7 @@ class HTTPBlobDownloader:
|
||||
log.debug("trying to download stream from mirror (sd %s)", self.sd_hashes[0][:8])
|
||||
else:
|
||||
log.debug("trying to download %i blobs from mirror", len(self.blob_hashes))
|
||||
blobs = yield DeferredDict(
|
||||
{blob_hash: self.blob_manager.get_blob(blob_hash) for blob_hash in self.blob_hashes}
|
||||
)
|
||||
blobs = {blob_hash: self.blob_manager.get_blob(blob_hash) for blob_hash in self.blob_hashes}
|
||||
self.deferreds = [self.download_blob(blobs[blob_hash]) for blob_hash in self.blob_hashes]
|
||||
yield defer.DeferredList(self.deferreds)
|
||||
if self.retry and self.missing_blob_hashes:
|
||||
@@ -175,7 +173,7 @@ class HTTPBlobDownloader:
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_stream(self, stream_hash, sd_hash):
|
||||
stream_crypt_blobs = yield self.blob_manager.storage.get_blobs_for_stream(stream_hash)
|
||||
stream_crypt_blobs = yield f2d(self.blob_manager.storage.get_blobs_for_stream(stream_hash))
|
||||
self.blob_hashes.extend([
|
||||
b.blob_hash for b in stream_crypt_blobs
|
||||
if b.blob_hash and b.blob_hash not in self.blob_hashes
|
||||
|
||||
@@ -2,8 +2,9 @@ import logging
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from twisted.internet import defer, threads, reactor
|
||||
from twisted.internet import defer, reactor
|
||||
|
||||
from lbrynet.extras.compat import d2f
|
||||
from lbrynet.blob.blob_file import BlobFile
|
||||
from lbrynet.p2p.BlobManager import DiskBlobManager
|
||||
from lbrynet.p2p.RateLimiter import DummyRateLimiter
|
||||
@@ -91,19 +92,16 @@ class SinglePeerDownloader:
|
||||
result = yield blob.callback
|
||||
if not result:
|
||||
log.debug("Failed to downloaded %s from %s", blob_hash[:16], peer.host)
|
||||
yield connection_manager.stop()
|
||||
yield connection_manager.stop()
|
||||
defer.returnValue(result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
|
||||
tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
|
||||
tmp_storage = SQLiteStorage(tmp_dir)
|
||||
yield tmp_storage.setup()
|
||||
async def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
|
||||
tmp_storage = SQLiteStorage(':memory:')
|
||||
await tmp_storage.open()
|
||||
tmp_dir = tempfile.mkdtemp()
|
||||
tmp_blob_manager = DiskBlobManager(tmp_dir, tmp_storage)
|
||||
try:
|
||||
result = yield self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager)
|
||||
return await d2f(self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager))
|
||||
finally:
|
||||
yield tmp_blob_manager.stop()
|
||||
yield tmp_storage.stop()
|
||||
yield threads.deferToThread(shutil.rmtree, tmp_dir)
|
||||
defer.returnValue(result)
|
||||
await tmp_storage.close()
|
||||
shutil.rmtree(tmp_dir)
|
||||
@@ -5,6 +5,7 @@ from collections import defaultdict
|
||||
from binascii import unhexlify
|
||||
from twisted.internet import threads, defer
|
||||
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.cryptoutils import get_lbry_hash_obj
|
||||
from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
|
||||
from lbrynet.p2p.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
|
||||
@@ -260,7 +261,7 @@ def save_sd_info(blob_manager, sd_hash, sd_info):
|
||||
if calculated_sd_hash != sd_hash:
|
||||
raise InvalidStreamDescriptorError("%s does not match calculated %s" %
|
||||
(sd_hash, calculated_sd_hash))
|
||||
stream_hash = yield blob_manager.storage.get_stream_hash_for_sd_hash(sd_hash)
|
||||
stream_hash = yield f2d(blob_manager.storage.get_stream_hash_for_sd_hash(sd_hash))
|
||||
if not stream_hash:
|
||||
log.debug("Saving info for %s", unhexlify(sd_info['stream_name']))
|
||||
stream_name = sd_info['stream_name']
|
||||
@@ -268,10 +269,10 @@ def save_sd_info(blob_manager, sd_hash, sd_info):
|
||||
stream_hash = sd_info['stream_hash']
|
||||
stream_blobs = sd_info['blobs']
|
||||
suggested_file_name = sd_info['suggested_file_name']
|
||||
yield blob_manager.storage.add_known_blobs(stream_blobs)
|
||||
yield blob_manager.storage.store_stream(
|
||||
yield f2d(blob_manager.storage.add_known_blobs(stream_blobs))
|
||||
yield f2d(blob_manager.storage.store_stream(
|
||||
stream_hash, sd_hash, stream_name, key, suggested_file_name, stream_blobs
|
||||
)
|
||||
))
|
||||
defer.returnValue(stream_hash)
|
||||
|
||||
|
||||
@@ -299,8 +300,7 @@ def format_sd_info(stream_type, stream_name, key, suggested_file_name, stream_ha
|
||||
}
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def get_sd_info(storage, stream_hash, include_blobs):
|
||||
async def get_sd_info(storage, stream_hash, include_blobs):
|
||||
"""
|
||||
Get an sd info dictionary from storage
|
||||
|
||||
@@ -329,16 +329,13 @@ def get_sd_info(storage, stream_hash, include_blobs):
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
stream_info = yield storage.get_stream_info(stream_hash)
|
||||
stream_info = await storage.get_stream_info(stream_hash)
|
||||
blobs = []
|
||||
if include_blobs:
|
||||
blobs = yield storage.get_blobs_for_stream(stream_hash)
|
||||
defer.returnValue(
|
||||
format_sd_info(
|
||||
EncryptedFileStreamType, stream_info[0], stream_info[1],
|
||||
stream_info[2], stream_hash, format_blobs(blobs)
|
||||
)
|
||||
blobs = await storage.get_blobs_for_stream(stream_hash)
|
||||
return format_sd_info(
|
||||
EncryptedFileStreamType, stream_info[0], stream_info[1],
|
||||
stream_info[2], stream_hash, format_blobs(blobs)
|
||||
)
|
||||
|
||||
|
||||
@@ -465,6 +462,6 @@ def download_sd_blob(blob_hash, blob_manager, peer_finder, rate_limiter, payment
|
||||
yield blob_manager.delete_blobs([blob_hash])
|
||||
raise err
|
||||
raw_sd = yield sd_reader._get_raw_data()
|
||||
yield blob_manager.storage.add_known_blob(blob_hash, len(raw_sd))
|
||||
yield f2d(blob_manager.storage.add_known_blob(blob_hash, len(raw_sd)))
|
||||
yield save_sd_info(blob_manager, sd_blob.blob_hash, sd_info)
|
||||
defer.returnValue(sd_blob)
|
||||
@@ -30,7 +30,7 @@ class DownloadManager:
|
||||
def resume_downloading(self):
|
||||
yield self.connection_manager.start()
|
||||
yield self.progress_manager.start()
|
||||
defer.returnValue(True)
|
||||
return True
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop_downloading(self):
|
||||
@@ -39,32 +39,14 @@ class DownloadManager:
|
||||
defer.returnValue(True)
|
||||
|
||||
def add_blobs_to_download(self, blob_infos):
|
||||
|
||||
log.debug("Adding %s blobs to blobs", len(blob_infos))
|
||||
|
||||
def add_blob_to_list(blob, blob_num):
|
||||
self.blobs[blob_num] = blob
|
||||
log.debug(
|
||||
"Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_num)
|
||||
|
||||
def error_during_add(err):
|
||||
log.warning(
|
||||
"An error occurred adding the blob to blobs. Error:%s", err.getErrorMessage())
|
||||
return err
|
||||
|
||||
ds = []
|
||||
for blob_info in blob_infos:
|
||||
if not blob_info.blob_num in self.blobs:
|
||||
self.blob_infos[blob_info.blob_num] = blob_info
|
||||
log.debug(
|
||||
"Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
|
||||
d = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
d.addCallback(add_blob_to_list, blob_info.blob_num)
|
||||
d.addErrback(error_during_add)
|
||||
ds.append(d)
|
||||
|
||||
dl = defer.DeferredList(ds)
|
||||
return dl
|
||||
log.debug("Trying to get the blob associated with blob hash %s", blob_info.blob_hash)
|
||||
blob = self.blob_manager.get_blob(blob_info.blob_hash, blob_info.length)
|
||||
self.blobs[blob_info.blob_num] = blob
|
||||
log.debug("Added blob (hash: %s, number %s) to the list", blob.blob_hash, blob_info.blob_num)
|
||||
|
||||
def stream_position(self):
|
||||
return self.progress_manager.stream_position()
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import logging
|
||||
from twisted.internet import defer
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -39,19 +38,12 @@ class BlobAvailabilityHandler:
|
||||
def handle_queries(self, queries):
|
||||
if self.query_identifiers[0] in queries:
|
||||
log.info("Received the client's list of requested blobs")
|
||||
d = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
|
||||
def set_field(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
|
||||
d.addCallback(set_field)
|
||||
return d
|
||||
return defer.succeed({})
|
||||
available_blobs = self._get_available_blobs(queries[self.query_identifiers[0]])
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
return {'available_blobs': available_blobs}
|
||||
return {}
|
||||
|
||||
######### internal #########
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
|
||||
return d
|
||||
return self.blob_manager.completed_blobs(requested_blobs)
|
||||
@@ -61,21 +61,21 @@ class BlobRequestHandler:
|
||||
request_handler.register_blob_sender(self)
|
||||
|
||||
def handle_queries(self, queries):
|
||||
response = defer.succeed({})
|
||||
response = {}
|
||||
log.debug("Handle query: %s", str(queries))
|
||||
|
||||
if self.AVAILABILITY_QUERY in queries:
|
||||
self._blobs_requested = queries[self.AVAILABILITY_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_availability(r, self._blobs_requested))
|
||||
self._reply_to_availability(response, self._blobs_requested)
|
||||
if self.PAYMENT_RATE_QUERY in queries:
|
||||
offered_rate = queries[self.PAYMENT_RATE_QUERY]
|
||||
offer = Offer(offered_rate)
|
||||
if offer.rate is None:
|
||||
log.warning("Empty rate offer")
|
||||
response.addCallback(lambda r: self._handle_payment_rate_query(offer, r))
|
||||
self._handle_payment_rate_query(offer, response)
|
||||
if self.BLOB_QUERY in queries:
|
||||
incoming = queries[self.BLOB_QUERY]
|
||||
response.addCallback(lambda r: self._reply_to_send_request(r, incoming))
|
||||
self._reply_to_send_request(response, incoming)
|
||||
return response
|
||||
|
||||
######### IBlobSender #########
|
||||
@@ -95,15 +95,10 @@ class BlobRequestHandler:
|
||||
######### internal #########
|
||||
|
||||
def _reply_to_availability(self, request, blobs):
|
||||
d = self._get_available_blobs(blobs)
|
||||
|
||||
def set_available(available_blobs):
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
request.update({'available_blobs': available_blobs})
|
||||
return request
|
||||
|
||||
d.addCallback(set_available)
|
||||
return d
|
||||
available_blobs = self._get_available_blobs(blobs)
|
||||
log.debug("available blobs: %s", str(available_blobs))
|
||||
request.update({'available_blobs': available_blobs})
|
||||
return request
|
||||
|
||||
def _handle_payment_rate_query(self, offer, request):
|
||||
blobs = self._blobs_requested
|
||||
@@ -172,8 +167,7 @@ class BlobRequestHandler:
|
||||
return d
|
||||
|
||||
def _get_available_blobs(self, requested_blobs):
|
||||
d = self.blob_manager.completed_blobs(requested_blobs)
|
||||
return d
|
||||
return self.blob_manager.completed_blobs(requested_blobs)
|
||||
|
||||
def send_file(self, consumer):
|
||||
|
||||
|
||||
@@ -167,7 +167,7 @@ class ServerRequestHandler:
|
||||
ds = []
|
||||
for query_handler, query_identifiers in self.query_handlers.items():
|
||||
queries = {q_i: msg[q_i] for q_i in query_identifiers if q_i in msg}
|
||||
d = query_handler.handle_queries(queries)
|
||||
d = defer.maybeDeferred(query_handler.handle_queries(queries))
|
||||
d.addErrback(log_errors)
|
||||
ds.append(d)
|
||||
|
||||
|
||||
+30
-25
@@ -7,19 +7,21 @@ from collections import OrderedDict
|
||||
|
||||
from lbrynet.schema.schema.claim import Claim
|
||||
from lbrynet.schema.proto import claim_pb2
|
||||
from lbrynet.schema.signature import Signature
|
||||
from lbrynet.schema.validator import get_validator
|
||||
from lbrynet.schema.signer import get_signer
|
||||
from lbrynet.schema.schema import NIST256p, CURVE_NAMES, CLAIM_TYPE_NAMES
|
||||
from lbrynet.schema.schema import NIST256p, CURVE_NAMES, CLAIM_TYPE_NAMES, SECP256k1
|
||||
from lbrynet.schema.encoding import decode_fields, decode_b64_fields, encode_fields
|
||||
from lbrynet.schema.error import DecodeError
|
||||
from lbrynet.schema.fee import Fee
|
||||
|
||||
|
||||
class ClaimDict(OrderedDict):
|
||||
def __init__(self, claim_dict=None):
|
||||
def __init__(self, claim_dict=None, detached_signature: Signature=None):
|
||||
if isinstance(claim_dict, claim_pb2.Claim):
|
||||
raise Exception("To initialize %s with a Claim protobuf use %s.load_protobuf" %
|
||||
(self.__class__.__name__, self.__class__.__name__))
|
||||
self.detached_signature = detached_signature
|
||||
OrderedDict.__init__(self, claim_dict or [])
|
||||
|
||||
@property
|
||||
@@ -37,7 +39,8 @@ class ClaimDict(OrderedDict):
|
||||
@property
|
||||
def serialized(self):
|
||||
"""Serialized Claim protobuf"""
|
||||
|
||||
if self.detached_signature:
|
||||
return self.detached_signature.serialized
|
||||
return self.protobuf.SerializeToString()
|
||||
|
||||
@property
|
||||
@@ -49,10 +52,9 @@ class ClaimDict(OrderedDict):
|
||||
|
||||
@property
|
||||
def has_signature(self):
|
||||
claim = self.protobuf
|
||||
if claim.HasField("publisherSignature"):
|
||||
return True
|
||||
return False
|
||||
return self.protobuf.HasField("publisherSignature") or (
|
||||
self.detached_signature and self.detached_signature.raw_signature
|
||||
)
|
||||
|
||||
@property
|
||||
def is_certificate(self):
|
||||
@@ -90,10 +92,11 @@ class ClaimDict(OrderedDict):
|
||||
return None
|
||||
|
||||
@property
|
||||
def certificate_id(self):
|
||||
if not self.has_signature:
|
||||
return None
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.certificateId)
|
||||
def certificate_id(self) -> str:
|
||||
if self.protobuf.HasField("publisherSignature"):
|
||||
return binascii.hexlify(self.protobuf.publisherSignature.certificateId).decode()
|
||||
if self.detached_signature and self.detached_signature.certificate_id:
|
||||
return binascii.hexlify(self.detached_signature.certificate_id).decode()
|
||||
|
||||
@property
|
||||
def signature(self):
|
||||
@@ -117,57 +120,59 @@ class ClaimDict(OrderedDict):
|
||||
def claim_dict(self):
|
||||
"""Claim dictionary with bytes represented as hex and base58"""
|
||||
|
||||
return dict(encode_fields(self))
|
||||
return dict(encode_fields(self, self.detached_signature))
|
||||
|
||||
@classmethod
|
||||
def load_protobuf_dict(cls, protobuf_dict):
|
||||
def load_protobuf_dict(cls, protobuf_dict, detached_signature=None):
|
||||
"""
|
||||
Load a ClaimDict from a dictionary with base64 encoded bytes
|
||||
(as returned by the protobuf json formatter)
|
||||
"""
|
||||
|
||||
return cls(decode_b64_fields(protobuf_dict))
|
||||
return cls(decode_b64_fields(protobuf_dict), detached_signature=detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_protobuf(cls, protobuf_claim):
|
||||
def load_protobuf(cls, protobuf_claim, detached_signature=None):
|
||||
"""Load ClaimDict from a protobuf Claim message"""
|
||||
return cls.load_protobuf_dict(json.loads(json_format.MessageToJson(protobuf_claim, True)))
|
||||
return cls.load_protobuf_dict(json.loads(json_format.MessageToJson(protobuf_claim, True)), detached_signature)
|
||||
|
||||
@classmethod
|
||||
def load_dict(cls, claim_dict):
|
||||
"""Load ClaimDict from a dictionary with hex and base58 encoded bytes"""
|
||||
try:
|
||||
return cls.load_protobuf(cls(decode_fields(claim_dict)).protobuf)
|
||||
claim_dict, detached_signature = decode_fields(claim_dict)
|
||||
return cls.load_protobuf(cls(claim_dict).protobuf, detached_signature)
|
||||
except json_format.ParseError as err:
|
||||
raise DecodeError(str(err))
|
||||
|
||||
@classmethod
|
||||
def deserialize(cls, serialized):
|
||||
"""Load a ClaimDict from a serialized protobuf string"""
|
||||
detached_signature = Signature.flagged_parse(serialized)
|
||||
|
||||
temp_claim = claim_pb2.Claim()
|
||||
try:
|
||||
temp_claim.ParseFromString(serialized)
|
||||
temp_claim.ParseFromString(detached_signature.payload)
|
||||
except DecodeError_pb:
|
||||
raise DecodeError(DecodeError_pb)
|
||||
return cls.load_protobuf(temp_claim)
|
||||
return cls.load_protobuf(temp_claim, detached_signature=detached_signature)
|
||||
|
||||
@classmethod
|
||||
def generate_certificate(cls, private_key, curve=NIST256p):
|
||||
def generate_certificate(cls, private_key, curve=SECP256k1):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
return cls.load_protobuf(signer.certificate)
|
||||
|
||||
def sign(self, private_key, claim_address, cert_claim_id, curve=NIST256p):
|
||||
def sign(self, private_key, claim_address, cert_claim_id, curve=SECP256k1, name=None, force_detached=False):
|
||||
signer = get_signer(curve).load_pem(private_key)
|
||||
signed = signer.sign_stream_claim(self, claim_address, cert_claim_id)
|
||||
return ClaimDict.load_protobuf(signed)
|
||||
signed, signature = signer.sign_stream_claim(self, claim_address, cert_claim_id, name, force_detached)
|
||||
return ClaimDict.load_protobuf(signed, signature)
|
||||
|
||||
def validate_signature(self, claim_address, certificate):
|
||||
def validate_signature(self, claim_address, certificate, name=None):
|
||||
if isinstance(certificate, ClaimDict):
|
||||
certificate = certificate.protobuf
|
||||
curve = CURVE_NAMES[certificate.certificate.keyType]
|
||||
validator = get_validator(curve).load_from_certificate(certificate, self.certificate_id)
|
||||
return validator.validate_claim_signature(self, claim_address)
|
||||
return validator.validate_claim_signature(self, claim_address, name)
|
||||
|
||||
def validate_private_key(self, private_key, certificate_id):
|
||||
certificate = self.protobuf
|
||||
|
||||
@@ -4,9 +4,10 @@ from lbrynet.schema.address import decode_address, encode_address
|
||||
from lbrynet.schema.schema import CLAIM_TYPES, CLAIM_TYPE, STREAM_TYPE, CERTIFICATE_TYPE
|
||||
from lbrynet.schema.schema import SIGNATURE
|
||||
from lbrynet.schema.error import DecodeError, InvalidAddress
|
||||
from lbrynet.schema.signature import Signature
|
||||
|
||||
|
||||
def encode_fields(claim_dictionary):
|
||||
def encode_fields(claim_dictionary, detached_signature: Signature):
|
||||
"""Encode bytes to hex and b58 for return by ClaimDict"""
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
@@ -27,12 +28,19 @@ def encode_fields(claim_dictionary):
|
||||
encoded_cert_id = binascii.hexlify(claim_dictionary[SIGNATURE]['certificateId']).decode()
|
||||
claim_dictionary[SIGNATURE]['signature'] = encoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = encoded_cert_id
|
||||
elif detached_signature and detached_signature.raw_signature:
|
||||
claim_dictionary[SIGNATURE] = {
|
||||
'detached_signature': binascii.hexlify(detached_signature.serialized).decode(),
|
||||
'certificateId': binascii.hexlify(detached_signature.certificate_id).decode()
|
||||
}
|
||||
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
|
||||
|
||||
def decode_fields(claim_dictionary):
|
||||
"""Decode hex and b58 encoded bytes in dictionaries given to ClaimDict"""
|
||||
detached_signature = None
|
||||
claim_dictionary = deepcopy(claim_dictionary)
|
||||
claim_type = CLAIM_TYPES[claim_dictionary[CLAIM_TYPE]]
|
||||
claim_value = claim_dictionary[claim_type]
|
||||
@@ -47,13 +55,17 @@ def decode_fields(claim_dictionary):
|
||||
elif claim_type == CLAIM_TYPES[CERTIFICATE_TYPE]:
|
||||
public_key = binascii.unhexlify(claim_value["publicKey"])
|
||||
claim_value["publicKey"] = public_key
|
||||
if SIGNATURE in claim_dictionary:
|
||||
if SIGNATURE in claim_dictionary and not claim_dictionary[SIGNATURE].get('detached_signature'):
|
||||
decoded_sig = binascii.unhexlify(claim_dictionary[SIGNATURE]['signature'])
|
||||
decoded_cert_id = binascii.unhexlify(claim_dictionary[SIGNATURE]['certificateId'])
|
||||
claim_dictionary[SIGNATURE]['signature'] = decoded_sig
|
||||
claim_dictionary[SIGNATURE]['certificateId'] = decoded_cert_id
|
||||
elif claim_dictionary.get(SIGNATURE, {}).get('detached_signature'):
|
||||
hex_detached_signature = claim_dictionary[SIGNATURE]['detached_signature']
|
||||
detached_signature = Signature.flagged_parse(binascii.unhexlify(hex_detached_signature))
|
||||
del claim_dictionary[SIGNATURE]
|
||||
claim_dictionary[claim_type] = claim_value
|
||||
return claim_dictionary
|
||||
return claim_dictionary, detached_signature
|
||||
|
||||
|
||||
def decode_b64_fields(claim_dictionary):
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from collections import namedtuple
|
||||
|
||||
from typing import Union
|
||||
|
||||
LEGACY = namedtuple('Legacy', 'payload')
|
||||
NAMED_SECP256K1 = namedtuple('NamedSECP256k1', 'raw_signature certificate_id payload')
|
||||
FLAGS = {
|
||||
LEGACY: 0x80,
|
||||
NAMED_SECP256K1: 0x01
|
||||
}
|
||||
|
||||
class Signature:
|
||||
|
||||
def __init__(self, data: Union[LEGACY, NAMED_SECP256K1]):
|
||||
self.data = data
|
||||
|
||||
@property
|
||||
def payload(self):
|
||||
return self.data.payload
|
||||
|
||||
@property
|
||||
def certificate_id(self):
|
||||
if type(self.data) == NAMED_SECP256K1:
|
||||
return self.data.certificate_id
|
||||
|
||||
@property
|
||||
def raw_signature(self):
|
||||
if type(self.data) == NAMED_SECP256K1:
|
||||
return self.data.raw_signature
|
||||
|
||||
@classmethod
|
||||
def flagged_parse(cls, binary: bytes):
|
||||
flag = binary[0]
|
||||
if flag == FLAGS[NAMED_SECP256K1]:
|
||||
return cls(NAMED_SECP256K1(binary[1:65], binary[65:85], binary[85:]))
|
||||
else:
|
||||
return cls(LEGACY(binary))
|
||||
|
||||
@property
|
||||
def serialized(self):
|
||||
if isinstance(self.data, NAMED_SECP256K1):
|
||||
return (bytes([FLAGS[type(self.data)]]) + self.data.raw_signature + self.data.certificate_id + self.payload)
|
||||
elif isinstance(self.data, LEGACY):
|
||||
return self.payload
|
||||
+29
-10
@@ -5,6 +5,7 @@ from lbrynet.schema.address import decode_address
|
||||
from lbrynet.schema.encoding import decode_b64_fields
|
||||
from lbrynet.schema.schema.certificate import Certificate
|
||||
from lbrynet.schema.schema.claim import Claim
|
||||
from lbrynet.schema.signature import Signature, NAMED_SECP256K1
|
||||
from lbrynet.schema.validator import validate_claim_id
|
||||
from lbrynet.schema.schema import V_0_0_1, CLAIM_TYPE, CLAIM_TYPES, CERTIFICATE_TYPE, VERSION
|
||||
from lbrynet.schema.schema import NIST256p, NIST384p, SECP256k1, SHA256, SHA384
|
||||
@@ -45,24 +46,41 @@ class NIST_ECDSASigner(object):
|
||||
def generate(cls):
|
||||
return cls(ecdsa.SigningKey.generate(curve=cls.CURVE, hashfunc=cls.HASHFUNC_NAME))
|
||||
|
||||
def sign_stream_claim(self, claim, claim_address, cert_claim_id):
|
||||
def sign(self, *fields):
|
||||
digest = self.HASHFUNC(bytearray(b''.join(fields))).digest()
|
||||
return self.private_key.sign_digest_deterministic(digest, hashfunc=self.HASHFUNC)
|
||||
|
||||
def sign_stream_claim(self, claim, claim_address, cert_claim_id, name, detached=False):
|
||||
validate_claim_id(cert_claim_id)
|
||||
raw_cert_id = binascii.unhexlify(cert_claim_id)
|
||||
decoded_addr = decode_address(claim_address)
|
||||
if detached:
|
||||
assert name, "Name is required for detached signatures"
|
||||
assert self.CURVE_NAME == SECP256k1, f"Only SECP256k1 is supported, not: {self.CURVE_NAME}"
|
||||
signature = self.sign(
|
||||
name.lower().encode(),
|
||||
decoded_addr,
|
||||
claim.serialized_no_signature,
|
||||
raw_cert_id,
|
||||
)
|
||||
else:
|
||||
signature = self.sign(decoded_addr, claim.serialized_no_signature, raw_cert_id)
|
||||
|
||||
to_sign = bytearray()
|
||||
to_sign.extend(decoded_addr)
|
||||
to_sign.extend(claim.serialized_no_signature)
|
||||
to_sign.extend(binascii.unhexlify(cert_claim_id))
|
||||
|
||||
digest = self.HASHFUNC(to_sign).digest()
|
||||
if detached:
|
||||
return Claim.load(decode_b64_fields(claim.protobuf_dict)), Signature(NAMED_SECP256K1(
|
||||
signature,
|
||||
raw_cert_id,
|
||||
claim.serialized_no_signature
|
||||
))
|
||||
# -- Legacy signer (signature inside protobuf) --
|
||||
|
||||
if not isinstance(self.private_key, ecdsa.SigningKey):
|
||||
raise Exception("Not given a signing key")
|
||||
sig_dict = {
|
||||
"version": V_0_0_1,
|
||||
"signatureType": self.CURVE_NAME,
|
||||
"signature": self.private_key.sign_digest_deterministic(digest, hashfunc=self.HASHFUNC),
|
||||
"certificateId": binascii.unhexlify(cert_claim_id)
|
||||
"signature": signature,
|
||||
"certificateId": raw_cert_id
|
||||
}
|
||||
|
||||
msg = {
|
||||
@@ -71,7 +89,8 @@ class NIST_ECDSASigner(object):
|
||||
"publisherSignature": sig_dict
|
||||
}
|
||||
|
||||
return Claim.load(msg)
|
||||
proto = Claim.load(msg)
|
||||
return proto, Signature.flagged_parse(proto.SerializeToString())
|
||||
|
||||
|
||||
class NIST256pSigner(NIST_ECDSASigner):
|
||||
|
||||
+13
-2
@@ -28,7 +28,7 @@ class URI(object):
|
||||
self.claim_id = claim_id
|
||||
self.path = path
|
||||
|
||||
if self.path is not None and not self.is_channel:
|
||||
if self.path is not None and not self.contains_channel:
|
||||
raise ValueError("Content claims cannot have paths")
|
||||
|
||||
def __str__(self):
|
||||
@@ -39,10 +39,21 @@ class URI(object):
|
||||
if not hasattr(other, prop) or getattr(self, prop) != getattr(other, prop):
|
||||
return False
|
||||
return self.__class__ == other.__class__
|
||||
@property
|
||||
def channel_name(self):
|
||||
return self.name if self.contains_channel else None
|
||||
|
||||
@property
|
||||
def claim_name(self):
|
||||
return self.name if not self.contains_channel else self.path
|
||||
|
||||
@property
|
||||
def contains_channel(self):
|
||||
return self.name.startswith(CHANNEL_CHAR)
|
||||
|
||||
@property
|
||||
def is_channel(self):
|
||||
return self.name.startswith(CHANNEL_CHAR)
|
||||
return self.contains_channel and not self.path
|
||||
|
||||
def to_uri_string(self):
|
||||
uri_string = PROTOCOL + "%s" % self.name
|
||||
|
||||
@@ -76,18 +76,25 @@ class Validator:
|
||||
from ecdsa import BadSignatureError
|
||||
raise BadSignatureError
|
||||
|
||||
def validate_claim_signature(self, claim, claim_address):
|
||||
def validate_claim_signature(self, claim, claim_address, name):
|
||||
to_sign = bytearray()
|
||||
if claim.detached_signature and claim.detached_signature.raw_signature:
|
||||
assert name is not None, "Name is required for verifying detached signatures."
|
||||
to_sign.extend(name.lower().encode())
|
||||
signature = claim.detached_signature.raw_signature
|
||||
payload = claim.detached_signature.payload
|
||||
else:
|
||||
# extract and serialize the stream from the claim, then check the signature
|
||||
signature = binascii.unhexlify(claim.signature)
|
||||
payload = claim.serialized_no_signature
|
||||
decoded_address = decode_address(claim_address)
|
||||
|
||||
# extract and serialize the stream from the claim, then check the signature
|
||||
signature = binascii.unhexlify(claim.signature)
|
||||
|
||||
if signature is None:
|
||||
raise Exception("No signature to validate")
|
||||
|
||||
to_sign = bytearray()
|
||||
to_sign.extend(decoded_address)
|
||||
to_sign.extend(claim.serialized_no_signature)
|
||||
to_sign.extend(payload)
|
||||
to_sign.extend(binascii.unhexlify(self.certificate_claim_id))
|
||||
|
||||
return self.validate_signature(self.HASHFUNC(to_sign).digest(), signature)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
FROM debian:buster-slim
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get upgrade -y && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
git \
|
||||
python3.7 \
|
||||
python3.7-dev \
|
||||
python3-pip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN python3.7 -m pip install --upgrade pip setuptools wheel
|
||||
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
|
||||
RUN python3.7 -m pip install git+https://github.com/lbryio/torba.git#egg=torba
|
||||
RUN python3.7 -m pip install -e .
|
||||
|
||||
# Orchstr8 API
|
||||
EXPOSE 7954
|
||||
# Wallet Server
|
||||
EXPOSE 5280
|
||||
# SPV Server
|
||||
EXPOSE 50002
|
||||
# blockchain
|
||||
EXPOSE 9246
|
||||
ENV TORBA_LEDGER lbrynet.extras.wallet
|
||||
|
||||
RUN /usr/local/bin/orchstr8 download
|
||||
ENTRYPOINT ["/usr/local/bin/orchstr8"]
|
||||
@@ -0,0 +1,49 @@
|
||||
FROM debian:buster-slim
|
||||
|
||||
ARG user=lbry
|
||||
|
||||
# create an unprivileged user
|
||||
RUN groupadd -g 999 $user && useradd -r -u 999 -g $user $user
|
||||
RUN mkdir -p /home/$user/
|
||||
RUN chown -R $user:$user /home/$user/
|
||||
|
||||
# install python, pip, git and clean up
|
||||
RUN apt-get update && apt-get -y --no-install-recommends install build-essential git python3.7 python3.7-dev python3-pip && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# create and chown database dir
|
||||
ARG db_dir=/database
|
||||
RUN mkdir -p $db_dir
|
||||
RUN chown -R $user:$user $db_dir
|
||||
|
||||
# change into our user
|
||||
USER $user
|
||||
WORKDIR /home/$user
|
||||
|
||||
# RUN pip3 install -U pip (broken on pip 10 https://github.com/pypa/pip/issues/5240)
|
||||
RUN python3.7 -m pip install --upgrade pip setuptools
|
||||
|
||||
# get torba
|
||||
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git#egg=torba
|
||||
|
||||
# copy lbrynet
|
||||
RUN mkdir projects/
|
||||
COPY . projects/lbry
|
||||
USER root
|
||||
RUN chown -R $user:$user .
|
||||
USER $user
|
||||
WORKDIR projects/lbry
|
||||
RUN python3.7 -m pip install --user -e . && rm ~/.cache -rf
|
||||
|
||||
# entry point
|
||||
ARG host=0.0.0.0
|
||||
ARG tcp_port=50001
|
||||
ARG daemon_url=http://lbry:lbry@localhost:9245/
|
||||
VOLUME $db_dir
|
||||
ENV TCP_PORT=$tcp_port
|
||||
ENV HOST=$host
|
||||
ENV DAEMON_URL=$daemon_url
|
||||
ENV DB_DIRECTORY=$db_dir
|
||||
ENV BANDWIDTH_LIMIT=1000000000000000000000000000000000000000000
|
||||
ENV MAX_SESSIONS=1000000000
|
||||
ENV MAX_SEND=1000000000000000000
|
||||
ENTRYPOINT ["/home/lbry/.local/bin/torba-server"]
|
||||
@@ -54,7 +54,7 @@ def download_it(peer, timeout, blob_hash):
|
||||
tmp_blob_manager = DiskBlobManager(tmp_dir, storage)
|
||||
|
||||
config = {'auto_connect': True}
|
||||
config['lbryum_wallet_dir'] = tempfile.mkdtemp()
|
||||
config['wallet_dir'] = tempfile.mkdtemp()
|
||||
config['use_keyring'] = False
|
||||
config['blockchain_name'] = conf.settings['blockchain_name']
|
||||
config['lbryum_servers'] = []
|
||||
|
||||
+18
-1
@@ -53,6 +53,15 @@ def get_label(pr, prefix):
|
||||
return label
|
||||
|
||||
|
||||
BACKWARDS_INCOMPATIBLE = 'backwards-incompatible:'
|
||||
|
||||
|
||||
def get_backwards_incompatible(desc: str):
|
||||
for line in desc.splitlines():
|
||||
if line.startswith(BACKWARDS_INCOMPATIBLE):
|
||||
yield line[len(BACKWARDS_INCOMPATIBLE):]
|
||||
|
||||
|
||||
def get_previous_final(repo, current_release):
|
||||
assert current_release.rc is not None, "Need an rc to find the previous final release."
|
||||
previous = None
|
||||
@@ -129,13 +138,16 @@ def release(args):
|
||||
else:
|
||||
previous_release = repo.release_from_tag(current_version.tag)
|
||||
|
||||
incompats = []
|
||||
areas = {}
|
||||
for pr in gh.search_issues(f"merged:>={previous_release._json_data['created_at']} repo:lbryio/lbry"):
|
||||
for area_name in get_labels(pr, 'area'):
|
||||
area = areas.setdefault(area_name, [])
|
||||
type_label = get_label(pr, "type")
|
||||
for incompat in get_backwards_incompatible(pr.body):
|
||||
incompats.append(f' * [{area_name}] {incompat.strip()} ({pr.html_url})')
|
||||
if not (args.action == '*-rc' and type_label == 'fixup'):
|
||||
area.append(f' * [{type_label}] {pr.title} ({pr.html_url})')
|
||||
area.append(f' * [{type_label}] {pr.title} ({pr.html_url}) by {pr.user["login"]}')
|
||||
|
||||
area_names = list(areas.keys())
|
||||
area_names.sort()
|
||||
@@ -144,6 +156,11 @@ def release(args):
|
||||
w = lambda s: body.write(s+'\n')
|
||||
|
||||
w(f'## [{new_version}] - {date.today().isoformat()}')
|
||||
if incompats:
|
||||
w('')
|
||||
w(f'### Backwards Incompatible Changes')
|
||||
for incompat in incompats:
|
||||
w(incompat)
|
||||
for area in area_names:
|
||||
prs = areas[area]
|
||||
area = AREA_RENAME.get(area, area.capitalize())
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import time
|
||||
|
||||
from aiohttp import ClientConnectorError
|
||||
from lbrynet import conf
|
||||
from lbrynet.schema.uri import parse_lbry_uri
|
||||
from lbrynet.extras.daemon.DaemonConsole import LBRYAPIClient
|
||||
|
||||
|
||||
def extract_uris(response):
|
||||
uris = list()
|
||||
for key in response:
|
||||
for value in response[key]:
|
||||
uris.append(value)
|
||||
|
||||
return uris
|
||||
|
||||
|
||||
async def get_frontpage_uris():
|
||||
session = aiohttp.ClientSession()
|
||||
try:
|
||||
response = await session.get("https://api.lbry.io/file/list_homepage", timeout=10.0)
|
||||
if response.status != 200:
|
||||
print("API returned non 200 code!!")
|
||||
return
|
||||
body = await response.json()
|
||||
await session.close()
|
||||
uris = extract_uris(body['data']['Uris'])
|
||||
return uris
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def main():
|
||||
uris = await get_frontpage_uris()
|
||||
print("got %i uris" % len(uris))
|
||||
api = await LBRYAPIClient.get_client()
|
||||
|
||||
try:
|
||||
await api.status()
|
||||
except (ClientConnectorError, ConnectionError):
|
||||
await api.session.close()
|
||||
print("Could not connect to daemon. Are you sure it's running?")
|
||||
return
|
||||
|
||||
first_byte_times = []
|
||||
|
||||
for uri in uris:
|
||||
await api.call(
|
||||
"file_delete", {
|
||||
"delete_from_download_dir": True,
|
||||
"delete_all": True,
|
||||
"claim_name": parse_lbry_uri(uri).name
|
||||
}
|
||||
)
|
||||
|
||||
for i, uri in enumerate(uris):
|
||||
start = time.time()
|
||||
try:
|
||||
await api.call("get", {"uri": uri})
|
||||
first_byte = time.time()
|
||||
first_byte_times.append(first_byte - start)
|
||||
print(f"{i + 1}/{len(uris)} - {first_byte - start} {uri}")
|
||||
except:
|
||||
print(f"{i + 1}/{len(uris)} - timed out in {time.time() - start} {uri}")
|
||||
await api.call(
|
||||
"file_delete", {
|
||||
"delete_from_download_dir": True,
|
||||
"claim_name": parse_lbry_uri(uri).name
|
||||
}
|
||||
)
|
||||
|
||||
avg = sum(first_byte_times) / len(first_byte_times)
|
||||
print()
|
||||
print(f"Average time to first byte: {avg} ({len(first_byte_times)} streams)")
|
||||
print(f"Started {len(first_byte_times)} Timed out {len(uris) - len(first_byte_times)}")
|
||||
|
||||
await api.session.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--data_dir", default=None)
|
||||
parser.add_argument("--wallet_dir", default=None)
|
||||
parser.add_argument("--download_directory", default=None)
|
||||
args = parser.parse_args()
|
||||
conf.initialize_settings(
|
||||
data_dir=args.data_dir, wallet_dir=args.wallet_dir, download_dir=args.download_directory
|
||||
)
|
||||
asyncio.run(main())
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
from unittest import skip
|
||||
from hashlib import md5
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.trial import unittest
|
||||
@@ -16,6 +17,7 @@ from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.blob.EncryptedFileCreator import create_lbry_file
|
||||
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
|
||||
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
|
||||
from lbrynet.extras.compat import f2d
|
||||
|
||||
from tests import mocks
|
||||
from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir
|
||||
@@ -70,15 +72,15 @@ class LbryUploader:
|
||||
if self.ul_rate_limit is not None:
|
||||
self.rate_limiter.set_ul_limit(self.ul_rate_limit)
|
||||
self.prm = OnlyFreePaymentsManager()
|
||||
self.storage = SQLiteStorage(self.db_dir)
|
||||
self.storage = SQLiteStorage(':memory:')
|
||||
self.blob_manager = DiskBlobManager(self.blob_dir, self.storage)
|
||||
self.lbry_file_manager = EncryptedFileManager(FakePeerFinder(5553, self.peer_manager, 1), self.rate_limiter,
|
||||
self.blob_manager, self.wallet, self.prm, self.storage,
|
||||
StreamDescriptorIdentifier())
|
||||
|
||||
yield self.storage.setup()
|
||||
yield self.blob_manager.setup()
|
||||
yield self.lbry_file_manager.setup()
|
||||
yield f2d(self.storage.open())
|
||||
yield f2d(self.blob_manager.setup())
|
||||
yield f2d(self.lbry_file_manager.setup())
|
||||
|
||||
query_handler_factories = {
|
||||
1: BlobAvailabilityHandlerFactory(self.blob_manager),
|
||||
@@ -103,15 +105,17 @@ class LbryUploader:
|
||||
for lbry_file in lbry_files:
|
||||
yield self.lbry_file_manager.delete_lbry_file(lbry_file)
|
||||
yield self.lbry_file_manager.stop()
|
||||
yield self.blob_manager.stop()
|
||||
yield self.storage.stop()
|
||||
yield f2d(self.blob_manager.stop())
|
||||
yield f2d(self.storage.close())
|
||||
self.server_port.stopListening()
|
||||
rm_db_and_blob_dir(self.db_dir, self.blob_dir)
|
||||
if os.path.exists("test_file"):
|
||||
os.remove("test_file")
|
||||
|
||||
|
||||
@skip
|
||||
class TestTransfer(unittest.TestCase):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setUp(self):
|
||||
mocks.mock_conf_settings(self)
|
||||
@@ -121,7 +125,7 @@ class TestTransfer(unittest.TestCase):
|
||||
self.peer_finder = FakePeerFinder(5553, self.peer_manager, 1)
|
||||
self.rate_limiter = RateLimiter()
|
||||
self.prm = OnlyFreePaymentsManager()
|
||||
self.storage = SQLiteStorage(self.db_dir)
|
||||
self.storage = SQLiteStorage(':memory:')
|
||||
self.blob_manager = DiskBlobManager(self.blob_dir, self.storage)
|
||||
self.sd_identifier = StreamDescriptorIdentifier()
|
||||
self.lbry_file_manager = EncryptedFileManager(self.peer_finder, self.rate_limiter,
|
||||
@@ -130,9 +134,9 @@ class TestTransfer(unittest.TestCase):
|
||||
|
||||
self.uploader = LbryUploader(5209343)
|
||||
self.sd_hash = yield self.uploader.setup()
|
||||
yield self.storage.setup()
|
||||
yield self.blob_manager.setup()
|
||||
yield self.lbry_file_manager.setup()
|
||||
yield f2d(self.storage.open())
|
||||
yield f2d(self.blob_manager.setup())
|
||||
yield f2d(self.lbry_file_manager.setup())
|
||||
yield add_lbry_file_to_sd_identifier(self.sd_identifier)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -143,7 +147,7 @@ class TestTransfer(unittest.TestCase):
|
||||
yield self.lbry_file_manager.delete_lbry_file(lbry_file)
|
||||
yield self.lbry_file_manager.stop()
|
||||
yield self.blob_manager.stop()
|
||||
yield self.storage.stop()
|
||||
yield f2d(self.storage.close())
|
||||
rm_db_and_blob_dir(self.db_dir, self.blob_dir)
|
||||
if os.path.exists("test_file"):
|
||||
os.remove("test_file")
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import os
|
||||
from unittest import skip
|
||||
from binascii import hexlify
|
||||
|
||||
from twisted.internet import defer, error
|
||||
from twisted.trial import unittest
|
||||
from lbrynet.p2p.StreamDescriptor import get_sd_info
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
|
||||
from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory
|
||||
from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory
|
||||
@@ -19,7 +21,10 @@ from tests import mocks
|
||||
from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir
|
||||
|
||||
|
||||
@skip
|
||||
class TestReflector(unittest.TestCase):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setUp(self):
|
||||
self.reflector_port = None
|
||||
self.port = None
|
||||
@@ -30,9 +35,9 @@ class TestReflector(unittest.TestCase):
|
||||
wallet = mocks.Wallet()
|
||||
peer_manager = PeerManager()
|
||||
peer_finder = mocks.PeerFinder(5553, peer_manager, 2)
|
||||
self.server_storage = SQLiteStorage(self.server_db_dir)
|
||||
self.server_storage = SQLiteStorage(':memory:')
|
||||
self.server_blob_manager = BlobManager.DiskBlobManager(self.server_blob_dir, self.server_storage)
|
||||
self.client_storage = SQLiteStorage(self.client_db_dir)
|
||||
self.client_storage = SQLiteStorage(':memory:')
|
||||
self.client_blob_manager = BlobManager.DiskBlobManager(self.client_blob_dir, self.client_storage)
|
||||
self.server_lbry_file_manager = EncryptedFileManager(
|
||||
peer_finder, DummyRateLimiter(), self.server_blob_manager, wallet, prm, self.server_storage,
|
||||
@@ -61,17 +66,17 @@ class TestReflector(unittest.TestCase):
|
||||
),
|
||||
]
|
||||
|
||||
d = self.server_storage.setup()
|
||||
d.addCallback(lambda _: self.server_blob_manager.setup())
|
||||
d.addCallback(lambda _: self.server_lbry_file_manager.setup())
|
||||
d.addCallback(lambda _: self.client_storage.setup())
|
||||
d.addCallback(lambda _: self.client_blob_manager.setup())
|
||||
d.addCallback(lambda _: self.client_lbry_file_manager.setup())
|
||||
yield f2d(self.server_storage.open())
|
||||
yield f2d(self.server_blob_manager.setup())
|
||||
yield f2d(self.server_lbry_file_manager.setup())
|
||||
yield f2d(self.client_storage.open())
|
||||
yield f2d(self.client_blob_manager.setup())
|
||||
yield f2d(self.client_lbry_file_manager.setup())
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def verify_equal(sd_info, stream_hash):
|
||||
self.assertDictEqual(mocks.create_stream_sd_file, sd_info)
|
||||
sd_hash = yield self.client_storage.get_sd_blob_hash_for_stream(stream_hash)
|
||||
sd_hash = yield f2d(self.client_storage.get_sd_blob_hash_for_stream(stream_hash))
|
||||
defer.returnValue(sd_hash)
|
||||
|
||||
def save_sd_blob_hash(sd_hash):
|
||||
@@ -80,7 +85,7 @@ class TestReflector(unittest.TestCase):
|
||||
|
||||
def verify_stream_descriptor_file(stream_hash):
|
||||
self.stream_hash = stream_hash
|
||||
d = get_sd_info(self.client_storage, stream_hash, True)
|
||||
d = f2d(get_sd_info(self.client_storage, stream_hash, True))
|
||||
d.addCallback(verify_equal, stream_hash)
|
||||
d.addCallback(save_sd_blob_hash)
|
||||
return d
|
||||
@@ -109,10 +114,9 @@ class TestReflector(unittest.TestCase):
|
||||
except error.CannotListenError:
|
||||
port += 1
|
||||
|
||||
d.addCallback(lambda _: create_stream())
|
||||
d.addCallback(verify_stream_descriptor_file)
|
||||
d.addCallback(lambda _: start_server())
|
||||
return d
|
||||
stream_hash = yield create_stream()
|
||||
yield verify_stream_descriptor_file(stream_hash)
|
||||
yield start_server()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def tearDown(self):
|
||||
@@ -120,15 +124,13 @@ class TestReflector(unittest.TestCase):
|
||||
for lbry_file in lbry_files:
|
||||
yield self.client_lbry_file_manager.delete_lbry_file(lbry_file)
|
||||
yield self.client_lbry_file_manager.stop()
|
||||
yield self.client_blob_manager.stop()
|
||||
yield self.client_storage.stop()
|
||||
yield f2d(self.client_storage.close())
|
||||
self.reflector_port.stopListening()
|
||||
lbry_files = self.server_lbry_file_manager.lbry_files
|
||||
for lbry_file in lbry_files:
|
||||
yield self.server_lbry_file_manager.delete_lbry_file(lbry_file)
|
||||
yield self.server_lbry_file_manager.stop()
|
||||
yield self.server_blob_manager.stop()
|
||||
yield self.server_storage.stop()
|
||||
yield f2d(self.server_storage.close())
|
||||
try:
|
||||
rm_db_and_blob_dir(self.client_db_dir, self.client_blob_dir)
|
||||
except Exception as err:
|
||||
@@ -150,15 +152,15 @@ class TestReflector(unittest.TestCase):
|
||||
@defer.inlineCallbacks
|
||||
def verify_stream_on_reflector():
|
||||
# check stream_info_manager has all the right information
|
||||
streams = yield self.server_storage.get_all_streams()
|
||||
streams = yield f2d(self.server_storage.get_all_streams())
|
||||
self.assertEqual(1, len(streams))
|
||||
self.assertEqual(self.stream_hash, streams[0])
|
||||
|
||||
blobs = yield self.server_storage.get_blobs_for_stream(self.stream_hash)
|
||||
blobs = yield f2d(self.server_storage.get_blobs_for_stream(self.stream_hash))
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
|
||||
expected_blob_hashes = [b[0] for b in self.expected_blobs[:-1] if b[0] is not None]
|
||||
self.assertEqual(expected_blob_hashes, blob_hashes)
|
||||
sd_hash = yield self.server_storage.get_sd_blob_hash_for_stream(streams[0])
|
||||
sd_hash = yield f2d(self.server_storage.get_sd_blob_hash_for_stream(streams[0]))
|
||||
self.assertEqual(self.sd_hash, sd_hash)
|
||||
|
||||
# check lbry file manager has the file
|
||||
@@ -166,14 +168,14 @@ class TestReflector(unittest.TestCase):
|
||||
|
||||
self.assertEqual(0, len(files))
|
||||
|
||||
streams = yield self.server_storage.get_all_streams()
|
||||
streams = yield f2d(self.server_storage.get_all_streams())
|
||||
self.assertEqual(1, len(streams))
|
||||
stream_info = yield self.server_storage.get_stream_info(self.stream_hash)
|
||||
stream_info = yield f2d(self.server_storage.get_stream_info(self.stream_hash))
|
||||
self.assertEqual(self.sd_hash, stream_info[3])
|
||||
self.assertEqual(hexlify(b'test_file').decode(), stream_info[0])
|
||||
|
||||
# check should_announce blobs on blob_manager
|
||||
blob_hashes = yield self.server_storage.get_all_should_announce_blobs()
|
||||
blob_hashes = yield f2d(self.server_storage.get_all_should_announce_blobs())
|
||||
self.assertSetEqual({self.sd_hash, expected_blob_hashes[0]}, set(blob_hashes))
|
||||
|
||||
def verify_have_blob(blob_hash, blob_size):
|
||||
@@ -232,10 +234,10 @@ class TestReflector(unittest.TestCase):
|
||||
@defer.inlineCallbacks
|
||||
def verify_stream_on_reflector():
|
||||
# this protocol should not have any impact on stream info manager
|
||||
streams = yield self.server_storage.get_all_streams()
|
||||
streams = yield f2d(self.server_storage.get_all_streams())
|
||||
self.assertEqual(0, len(streams))
|
||||
# there should be no should announce blobs here
|
||||
blob_hashes = yield self.server_storage.get_all_should_announce_blobs()
|
||||
blob_hashes = yield f2d(self.server_storage.get_all_should_announce_blobs())
|
||||
self.assertEqual(0, len(blob_hashes))
|
||||
|
||||
def verify_data_on_reflector():
|
||||
@@ -270,6 +272,7 @@ class TestReflector(unittest.TestCase):
|
||||
|
||||
# test case when we reflect blob, and than that same blob
|
||||
# is reflected as stream
|
||||
@defer.inlineCallbacks
|
||||
def test_blob_reflect_and_stream(self):
|
||||
|
||||
def verify_blob_on_reflector():
|
||||
@@ -282,20 +285,19 @@ class TestReflector(unittest.TestCase):
|
||||
def verify_stream_on_reflector():
|
||||
# check stream_info_manager has all the right information
|
||||
|
||||
streams = yield self.server_storage.get_all_streams()
|
||||
streams = yield f2d(self.server_storage.get_all_streams())
|
||||
self.assertEqual(1, len(streams))
|
||||
self.assertEqual(self.stream_hash, streams[0])
|
||||
|
||||
blobs = yield self.server_storage.get_blobs_for_stream(self.stream_hash)
|
||||
blobs = yield f2d(self.server_storage.get_blobs_for_stream(self.stream_hash))
|
||||
blob_hashes = [b.blob_hash for b in blobs if b.blob_hash is not None]
|
||||
expected_blob_hashes = [b[0] for b in self.expected_blobs[:-1] if b[0] is not None]
|
||||
self.assertEqual(expected_blob_hashes, blob_hashes)
|
||||
sd_hash = yield self.server_storage.get_sd_blob_hash_for_stream(
|
||||
self.stream_hash)
|
||||
sd_hash = yield f2d(self.server_storage.get_sd_blob_hash_for_stream(self.stream_hash))
|
||||
self.assertEqual(self.sd_hash, sd_hash)
|
||||
|
||||
# check should_announce blobs on blob_manager
|
||||
to_announce = yield self.server_storage.get_all_should_announce_blobs()
|
||||
to_announce = yield f2d(self.server_storage.get_all_should_announce_blobs())
|
||||
self.assertSetEqual(set(to_announce), {self.sd_hash, expected_blob_hashes[0]})
|
||||
|
||||
def verify_have_blob(blob_hash, blob_size):
|
||||
@@ -328,11 +330,10 @@ class TestReflector(unittest.TestCase):
|
||||
# Modify this to change which blobs to send
|
||||
blobs_to_send = self.expected_blobs
|
||||
|
||||
d = send_to_server_as_blobs([x[0] for x in self.expected_blobs])
|
||||
d.addCallback(send_to_server_as_stream)
|
||||
d.addCallback(lambda _: verify_blob_on_reflector())
|
||||
d.addCallback(lambda _: verify_stream_on_reflector())
|
||||
return d
|
||||
finished = yield send_to_server_as_blobs([x[0] for x in self.expected_blobs])
|
||||
yield send_to_server_as_stream(finished)
|
||||
yield verify_blob_on_reflector()
|
||||
yield verify_stream_on_reflector()
|
||||
|
||||
|
||||
def iv_generator():
|
||||
|
||||
@@ -8,6 +8,7 @@ from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier
|
||||
from lbrynet.p2p.BlobManager import DiskBlobManager
|
||||
from lbrynet.p2p.StreamDescriptor import get_sd_info
|
||||
from lbrynet.p2p.RateLimiter import DummyRateLimiter
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.daemon.PeerManager import PeerManager
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
|
||||
@@ -28,6 +29,7 @@ DummyBlobAvailabilityTracker = mocks.BlobAvailabilityTracker
|
||||
class TestStreamify(TestCase):
|
||||
maxDiff = 5000
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setUp(self):
|
||||
mocks.mock_conf_settings(self)
|
||||
self.session = None
|
||||
@@ -42,16 +44,15 @@ class TestStreamify(TestCase):
|
||||
self.peer_finder = FakePeerFinder(5553, self.peer_manager, 2)
|
||||
self.rate_limiter = DummyRateLimiter()
|
||||
self.sd_identifier = StreamDescriptorIdentifier()
|
||||
self.storage = SQLiteStorage(self.db_dir)
|
||||
self.storage = SQLiteStorage(':memory:')
|
||||
self.blob_manager = DiskBlobManager(self.blob_dir, self.storage, self.dht_node._dataStore)
|
||||
self.prm = OnlyFreePaymentsManager()
|
||||
self.lbry_file_manager = EncryptedFileManager(
|
||||
self.peer_finder, self.rate_limiter, self.blob_manager, self.wallet, self.prm, self.storage,
|
||||
self.sd_identifier
|
||||
)
|
||||
d = self.storage.setup()
|
||||
d.addCallback(lambda _: self.lbry_file_manager.setup())
|
||||
return d
|
||||
yield f2d(self.storage.open())
|
||||
yield f2d(self.lbry_file_manager.setup())
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def tearDown(self):
|
||||
@@ -59,8 +60,8 @@ class TestStreamify(TestCase):
|
||||
for lbry_file in lbry_files:
|
||||
yield self.lbry_file_manager.delete_lbry_file(lbry_file)
|
||||
yield self.lbry_file_manager.stop()
|
||||
yield self.storage.stop()
|
||||
yield threads.deferToThread(shutil.rmtree, self.db_dir)
|
||||
yield f2d(self.storage.close())
|
||||
shutil.rmtree(self.db_dir, ignore_errors=True)
|
||||
if os.path.exists("test_file"):
|
||||
os.remove("test_file")
|
||||
|
||||
@@ -70,7 +71,7 @@ class TestStreamify(TestCase):
|
||||
self.assertEqual(sd_info, test_create_stream_sd_file)
|
||||
|
||||
def verify_stream_descriptor_file(stream_hash):
|
||||
d = get_sd_info(self.storage, stream_hash, True)
|
||||
d = f2d(get_sd_info(self.storage, stream_hash, True))
|
||||
d.addCallback(verify_equal)
|
||||
return d
|
||||
|
||||
@@ -98,7 +99,7 @@ class TestStreamify(TestCase):
|
||||
test_file = GenFile(53209343, bytes((i + 5) for i in range(0, 64, 6)))
|
||||
lbry_file = yield create_lbry_file(self.blob_manager, self.storage, self.prm, self.lbry_file_manager,
|
||||
"test_file", test_file)
|
||||
sd_hash = yield self.storage.get_sd_blob_hash_for_stream(lbry_file.stream_hash)
|
||||
sd_hash = yield f2d(self.storage.get_sd_blob_hash_for_stream(lbry_file.stream_hash))
|
||||
self.assertTrue(lbry_file.sd_hash, sd_hash)
|
||||
yield lbry_file.start()
|
||||
f = open('test_file', 'rb')
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import contextlib
|
||||
from twisted.trial import unittest
|
||||
from io import StringIO
|
||||
from twisted.internet import defer
|
||||
from unittest import skip
|
||||
from torba.testcase import AsyncioTestCase
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras import cli
|
||||
@@ -18,21 +18,19 @@ class FakeAnalytics:
|
||||
def is_started(self):
|
||||
return True
|
||||
|
||||
def send_server_startup_success(self):
|
||||
async def send_server_startup_success(self):
|
||||
pass
|
||||
|
||||
def send_server_startup(self):
|
||||
async def send_server_startup(self):
|
||||
pass
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
|
||||
class CLIIntegrationTest(unittest.TestCase):
|
||||
USE_AUTH = False
|
||||
class CLIIntegrationTest(AsyncioTestCase):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def setUp(self):
|
||||
async def asyncSetUp(self):
|
||||
skip = [
|
||||
DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT,
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT,
|
||||
@@ -41,34 +39,16 @@ class CLIIntegrationTest(unittest.TestCase):
|
||||
]
|
||||
conf.initialize_settings(load_conf_file=False)
|
||||
conf.settings['api_port'] = 5299
|
||||
conf.settings['use_auth_http'] = self.USE_AUTH
|
||||
conf.settings['components_to_skip'] = skip
|
||||
conf.settings.initialize_post_conf_load()
|
||||
Daemon.component_attributes = {}
|
||||
self.daemon = Daemon(analytics_manager=FakeAnalytics())
|
||||
yield self.daemon.start_listening()
|
||||
await self.daemon.start_listening()
|
||||
|
||||
def tearDown(self):
|
||||
return self.daemon._shutdown()
|
||||
|
||||
|
||||
class AuthenticatedCLITest(CLIIntegrationTest):
|
||||
USE_AUTH = True
|
||||
async def asyncTearDown(self):
|
||||
await self.daemon.shutdown()
|
||||
|
||||
def test_cli_status_command_with_auth(self):
|
||||
self.assertTrue(self.daemon._use_authentication)
|
||||
actual_output = StringIO()
|
||||
with contextlib.redirect_stdout(actual_output):
|
||||
cli.main(["status"])
|
||||
actual_output = actual_output.getvalue()
|
||||
self.assertIn("connection_status", actual_output)
|
||||
|
||||
|
||||
class UnauthenticatedCLITest(CLIIntegrationTest):
|
||||
USE_AUTH = False
|
||||
|
||||
def test_cli_status_command_with_auth(self):
|
||||
self.assertFalse(self.daemon._use_authentication)
|
||||
actual_output = StringIO()
|
||||
with contextlib.redirect_stdout(actual_output):
|
||||
cli.main(["status"])
|
||||
|
||||
@@ -1,74 +1,30 @@
|
||||
import sys
|
||||
import json
|
||||
import asyncio
|
||||
import tempfile
|
||||
import logging
|
||||
from binascii import unhexlify
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
|
||||
from twisted.trial import unittest
|
||||
from twisted.internet import utils, defer
|
||||
from twisted.internet.utils import runWithWarningsSuppressed as originalRunWith
|
||||
import twisted.internet
|
||||
from twisted.internet.asyncioreactor import AsyncioSelectorReactor
|
||||
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
from lbrynet.p2p.Error import InsufficientFundsError
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
|
||||
from torba.testcase import IntegrationTestCase as BaseIntegrationTestCase
|
||||
from torba.testcase import IntegrationTestCase
|
||||
|
||||
import lbrynet.schema
|
||||
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
|
||||
|
||||
from lbrynet import conf as lbry_conf
|
||||
from lbrynet.dht.node import Node
|
||||
from lbrynet.extras.daemon.Daemon import Daemon
|
||||
from lbrynet.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from lbrynet.extras.daemon.Components import WalletComponent, DHTComponent, HashAnnouncerComponent, \
|
||||
ExchangeRateManagerComponent
|
||||
from lbrynet.extras.daemon.Components import REFLECTOR_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT
|
||||
from lbrynet.extras.daemon.Components import UPnPComponent
|
||||
from lbrynet.extras.daemon.Components import d2f
|
||||
from lbrynet.extras.daemon.Components import WalletComponent
|
||||
from lbrynet.extras.daemon.Components import (
|
||||
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
REFLECTOR_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
)
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.auth.server import jsonrpc_dumps_pretty
|
||||
|
||||
|
||||
class FakeUPnP(UPnPComponent):
|
||||
|
||||
def __init__(self, component_manager):
|
||||
self.component_manager = component_manager
|
||||
self._running = False
|
||||
self.use_upnp = False
|
||||
self.upnp_redirects = {}
|
||||
|
||||
def start(self):
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class FakeDHT(DHTComponent):
|
||||
|
||||
def start(self):
|
||||
self.dht_node = Node()
|
||||
|
||||
|
||||
class FakeExchangeRateComponent(ExchangeRateManagerComponent):
|
||||
|
||||
def start(self):
|
||||
self.exchange_rate_manager = SimpleNamespace()
|
||||
|
||||
def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class FakeHashAnnouncerComponent(HashAnnouncerComponent):
|
||||
|
||||
def start(self):
|
||||
self.hash_announcer = SimpleNamespace()
|
||||
|
||||
def stop(self):
|
||||
pass
|
||||
|
||||
|
||||
class FakeAnalytics:
|
||||
@@ -77,57 +33,41 @@ class FakeAnalytics:
|
||||
def is_started(self):
|
||||
return True
|
||||
|
||||
def send_new_channel(self):
|
||||
async def send_new_channel(self):
|
||||
pass
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
def send_claim_action(self, action):
|
||||
async def send_claim_action(self, action):
|
||||
pass
|
||||
|
||||
def send_credits_sent(self):
|
||||
async def send_credits_sent(self):
|
||||
pass
|
||||
|
||||
def send_server_startup(self):
|
||||
async def send_server_startup(self):
|
||||
pass
|
||||
|
||||
|
||||
class IntegrationTestCase(unittest.TestCase, BaseIntegrationTestCase):
|
||||
|
||||
async def setUp(self):
|
||||
await self.asyncSetUp()
|
||||
|
||||
async def tearDown(self):
|
||||
await self.asyncTearDown()
|
||||
|
||||
|
||||
def run_with_async_support(suppress, f, *a, **kw):
|
||||
if asyncio.iscoroutinefunction(f):
|
||||
def test_method(*args, **kwargs):
|
||||
return defer.Deferred.fromFuture(asyncio.ensure_future(f(*args, **kwargs)))
|
||||
else:
|
||||
test_method = f
|
||||
return originalRunWith(suppress, test_method, *a, **kw)
|
||||
|
||||
|
||||
utils.runWithWarningsSuppressed = run_with_async_support
|
||||
|
||||
|
||||
class CommandTestCase(IntegrationTestCase):
|
||||
|
||||
timeout = 180
|
||||
MANAGER = LbryWalletManager
|
||||
VERBOSITY = logging.WARN
|
||||
|
||||
async def setUp(self):
|
||||
await super().setUp()
|
||||
async def asyncSetUp(self):
|
||||
await super().asyncSetUp()
|
||||
|
||||
twisted.internet.reactor = sys.modules['twisted.internet.reactor'] = AsyncioSelectorReactor()
|
||||
|
||||
logging.getLogger('lbrynet.p2p').setLevel(self.VERBOSITY)
|
||||
logging.getLogger('lbrynet.daemon').setLevel(self.VERBOSITY)
|
||||
|
||||
lbry_conf.settings = None
|
||||
lbry_conf.initialize_settings(
|
||||
load_conf_file=False, data_dir=self.wallet_node.data_path, wallet_dir=self.wallet_node.data_path,
|
||||
load_conf_file=False,
|
||||
data_dir=self.wallet_node.data_path,
|
||||
wallet_dir=self.wallet_node.data_path,
|
||||
download_dir=self.wallet_node.data_path
|
||||
)
|
||||
lbry_conf.settings['use_upnp'] = False
|
||||
@@ -150,27 +90,22 @@ class CommandTestCase(IntegrationTestCase):
|
||||
return self.wallet_component
|
||||
|
||||
skip = [
|
||||
#UPNP_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT,
|
||||
REFLECTOR_COMPONENT
|
||||
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
]
|
||||
analytics_manager = FakeAnalytics()
|
||||
self.daemon = Daemon(analytics_manager, ComponentManager(
|
||||
analytics_manager=analytics_manager,
|
||||
skip_components=skip, wallet=wallet_maker,
|
||||
dht=FakeDHT, hash_announcer=FakeHashAnnouncerComponent,
|
||||
exchange_rate_manager=FakeExchangeRateComponent,
|
||||
upnp=FakeUPnP
|
||||
skip_components=skip, wallet=wallet_maker
|
||||
))
|
||||
await d2f(self.daemon.setup())
|
||||
await self.daemon.setup()
|
||||
self.daemon.wallet_manager = self.wallet_component.wallet_manager
|
||||
self.manager.old_db = self.daemon.storage
|
||||
|
||||
async def tearDown(self):
|
||||
self.conductor.spv_node.server.stop()
|
||||
await super().tearDown()
|
||||
async def asyncTearDown(self):
|
||||
await super().asyncTearDown()
|
||||
self.wallet_component._running = False
|
||||
await d2f(self.daemon._shutdown())
|
||||
await self.daemon.shutdown()
|
||||
|
||||
async def confirm_tx(self, txid):
|
||||
""" Wait for tx to be in mempool, then generate a block, wait for tx to be in a block. """
|
||||
@@ -661,6 +596,27 @@ class ClaimManagement(CommandTestCase):
|
||||
await self.out(self.daemon.jsonrpc_claim_abandon(claim['claim_id']))
|
||||
self.assertEqual('9.97968399', await self.daemon.jsonrpc_account_balance())
|
||||
|
||||
async def test_claim_show(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
await self.confirm_tx(channel['tx']['txid'])
|
||||
channel_from_claim_show = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(txid=channel['tx']['txid'], nout=channel['output']['nout'])
|
||||
)
|
||||
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
|
||||
channel_from_claim_show = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(claim_id=channel['claim_id'])
|
||||
)
|
||||
self.assertEqual(channel_from_claim_show['value'], channel['output']['value'])
|
||||
|
||||
abandon = await self.out(self.daemon.jsonrpc_claim_abandon(txid=channel['tx']['txid'], nout=0, blocking=False))
|
||||
self.assertTrue(abandon['success'])
|
||||
await self.confirm_tx(abandon['tx']['txid'])
|
||||
not_a_claim = await self.out(
|
||||
self.daemon.jsonrpc_claim_show(txid=abandon['tx']['txid'], nout=0)
|
||||
)
|
||||
self.assertEqual(not_a_claim, 'claim not found')
|
||||
|
||||
async def test_abandoned_channel_with_signed_claims(self):
|
||||
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
|
||||
self.assertTrue(channel['success'])
|
||||
@@ -846,3 +802,14 @@ class TransactionCommandsTestCase(CommandTestCase):
|
||||
# inexistent
|
||||
result = await self.daemon.jsonrpc_transaction_show('0'*64)
|
||||
self.assertFalse(result['success'])
|
||||
|
||||
async def test_utxo_release(self):
|
||||
sendtxid = await self.blockchain.send_to_address(
|
||||
await self.account.receiving.get_or_create_usable_address(), 1
|
||||
)
|
||||
await self.confirm_tx(sendtxid)
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
await self.ledger.reserve_outputs(await self.account.get_utxos())
|
||||
await self.assertBalance(self.account, '0.0')
|
||||
await self.daemon.jsonrpc_utxo_release()
|
||||
await self.assertBalance(self.account, '11.0')
|
||||
@@ -1,6 +1,11 @@
|
||||
import binascii
|
||||
import logging
|
||||
import asyncio
|
||||
|
||||
from lbrynet.schema.address import decode_address
|
||||
from lbrynet.schema.schema import SECP256k1
|
||||
from lbrynet.schema.signature import Signature, NAMED_SECP256K1
|
||||
from lbrynet.schema.signer import get_signer
|
||||
from torba.testcase import IntegrationTestCase
|
||||
from lbrynet.schema.claim import ClaimDict
|
||||
from lbrynet.extras.wallet.transaction import Transaction
|
||||
@@ -60,7 +65,7 @@ class BasicTransactionTest(IntegrationTestCase):
|
||||
cert, key = generate_certificate()
|
||||
cert_tx = await Transaction.claim('@bar', cert, l2d('1.0'), address1, [self.account], self.account)
|
||||
claim = ClaimDict.load_dict(example_claim_dict)
|
||||
claim = claim.sign(key, address1, cert_tx.outputs[0].claim_id)
|
||||
claim = claim.sign(key, address1, cert_tx.outputs[0].claim_id, name='foo')
|
||||
claim_tx = await Transaction.claim('foo', claim, l2d('1.0'), address1, [self.account], self.account)
|
||||
|
||||
await self.broadcast(cert_tx)
|
||||
@@ -95,3 +100,73 @@ class BasicTransactionTest(IntegrationTestCase):
|
||||
response = await self.ledger.resolve(0, 10, 'lbry://404', 'lbry://@404')
|
||||
self.assertEqual('URI lbry://404 cannot be resolved', response['lbry://404']['error'])
|
||||
self.assertEqual('URI lbry://@404 cannot be resolved', response['lbry://@404']['error'])
|
||||
|
||||
async def test_new_signature_model(self):
|
||||
address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True)
|
||||
sendtxid1 = await self.blockchain.send_to_address(address1, 5)
|
||||
sendtxid2 = await self.blockchain.send_to_address(address2, 5)
|
||||
await self.blockchain.generate(1)
|
||||
await asyncio.wait([
|
||||
self.on_transaction_id(sendtxid1),
|
||||
self.on_transaction_id(sendtxid2)
|
||||
])
|
||||
|
||||
self.assertEqual(d2l(await self.account.get_balance()), '10.0')
|
||||
|
||||
cert, key = generate_certificate()
|
||||
cert_tx = await Transaction.claim('@bar', cert, l2d('1.0'), address1, [self.account], self.account)
|
||||
claim = ClaimDict.load_dict(example_claim_dict)
|
||||
claim = claim.sign(key, address1, cert_tx.outputs[0].claim_id, name='foo', curve=SECP256k1, force_detached=True)
|
||||
claim_tx = await Transaction.claim('foo', claim, l2d('1.0'), address1, [self.account], self.account)
|
||||
|
||||
await self.broadcast(cert_tx)
|
||||
await self.broadcast(claim_tx)
|
||||
await self.ledger.wait(claim_tx)
|
||||
await self.blockchain.generate(1)
|
||||
await self.ledger.wait(claim_tx)
|
||||
|
||||
response = await self.ledger.resolve(0, 10, 'lbry://@bar/foo')
|
||||
self.assertIn('lbry://@bar/foo', response)
|
||||
self.assertIn('claim', response['lbry://@bar/foo'])
|
||||
|
||||
async def test_new_signature_model_from_unserializable_claim(self):
|
||||
address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True)
|
||||
sendtxid1 = await self.blockchain.send_to_address(address1, 5)
|
||||
sendtxid2 = await self.blockchain.send_to_address(address2, 5)
|
||||
await self.blockchain.generate(1)
|
||||
await asyncio.wait([
|
||||
self.on_transaction_id(sendtxid1),
|
||||
self.on_transaction_id(sendtxid2)
|
||||
])
|
||||
|
||||
self.assertEqual(d2l(await self.account.get_balance()), '10.0')
|
||||
|
||||
cert, key = generate_certificate()
|
||||
cert_tx = await Transaction.claim('@bar', cert, l2d('1.0'), address1, [self.account], self.account)
|
||||
original = ClaimDict.load_dict(example_claim_dict).serialized
|
||||
altered = original + b'\x00\x01\x02\x30\x50\x80\x99' # pretend this extra trash is from some unknown protobuf
|
||||
|
||||
# manually sign
|
||||
signer = get_signer(SECP256k1).load_pem(key)
|
||||
signature = signer.sign(
|
||||
b'foo',
|
||||
decode_address(address1),
|
||||
altered,
|
||||
binascii.unhexlify(cert_tx.outputs[0].claim_id),
|
||||
)
|
||||
detached_sig = Signature(NAMED_SECP256K1(
|
||||
signature,
|
||||
binascii.unhexlify(cert_tx.outputs[0].claim_id),
|
||||
altered
|
||||
))
|
||||
claim_tx = await Transaction.claim('foo', detached_sig, l2d('1.0'), address1, [self.account], self.account)
|
||||
|
||||
await self.broadcast(cert_tx)
|
||||
await self.broadcast(claim_tx)
|
||||
await self.ledger.wait(claim_tx)
|
||||
await self.blockchain.generate(1)
|
||||
await self.ledger.wait(claim_tx)
|
||||
|
||||
response = await self.ledger.resolve(0, 10, 'lbry://@bar/foo')
|
||||
self.assertIn('lbry://@bar/foo', response)
|
||||
self.assertIn('claim', response['lbry://@bar/foo'])
|
||||
+26
-48
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
from unittest import mock
|
||||
@@ -151,16 +152,15 @@ class PointTraderKeyQueryHandler:
|
||||
try:
|
||||
decode_rsa_key(new_encoded_pub_key)
|
||||
except (ValueError, TypeError, IndexError):
|
||||
value_error = ValueError(f"Client sent an invalid public key: {new_encoded_pub_key}")
|
||||
return defer.fail(Failure(value_error))
|
||||
raise ValueError(f"Client sent an invalid public key: {new_encoded_pub_key}")
|
||||
self.public_key = new_encoded_pub_key
|
||||
self.wallet.set_public_key_for_peer(self.peer, self.public_key)
|
||||
fields = {'public_key': self.wallet.encoded_public_key.decode()}
|
||||
return defer.succeed(fields)
|
||||
return fields
|
||||
if self.public_key is None:
|
||||
return defer.fail(Failure(ValueError("Expected but did not receive a public key")))
|
||||
raise ValueError("Expected but did not receive a public key")
|
||||
else:
|
||||
return defer.succeed({})
|
||||
return {}
|
||||
|
||||
|
||||
class Wallet:
|
||||
@@ -377,27 +377,28 @@ class FakeComponent:
|
||||
def running(self):
|
||||
return self._running
|
||||
|
||||
def start(self):
|
||||
raise NotImplementedError # Override
|
||||
async def start(self):
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(None)
|
||||
async def stop(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def component(self):
|
||||
return self
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _setup(self):
|
||||
result = yield defer.maybeDeferred(self.start)
|
||||
async def _setup(self):
|
||||
result = await self.start()
|
||||
self._running = True
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _stop(self):
|
||||
result = yield defer.maybeDeferred(self.stop)
|
||||
async def _stop(self):
|
||||
result = await self.stop()
|
||||
self._running = False
|
||||
defer.returnValue(result)
|
||||
return result
|
||||
|
||||
async def get_status(self):
|
||||
return {}
|
||||
|
||||
def __lt__(self, other):
|
||||
return self.component_name < other.component_name
|
||||
@@ -407,41 +408,27 @@ class FakeDelayedWallet(FakeComponent):
|
||||
component_name = "wallet"
|
||||
depends_on = []
|
||||
|
||||
def start(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
d = defer.Deferred()
|
||||
self.component_manager.reactor.callLater(1, d.callback, True)
|
||||
return d
|
||||
async def stop(self):
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
class FakeDelayedBlobManager(FakeComponent):
|
||||
component_name = "blob_manager"
|
||||
depends_on = [FakeDelayedWallet.component_name]
|
||||
|
||||
def start(self):
|
||||
d = defer.Deferred()
|
||||
self.component_manager.reactor.callLater(1, d.callback, True)
|
||||
return d
|
||||
async def start(self):
|
||||
await asyncio.sleep(1)
|
||||
|
||||
def stop(self):
|
||||
d = defer.Deferred()
|
||||
self.component_manager.reactor.callLater(1, d.callback, True)
|
||||
return d
|
||||
async def stop(self):
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
class FakeDelayedFileManager(FakeComponent):
|
||||
component_name = "file_manager"
|
||||
depends_on = [FakeDelayedBlobManager.component_name]
|
||||
|
||||
def start(self):
|
||||
d = defer.Deferred()
|
||||
self.component_manager.reactor.callLater(1, d.callback, True)
|
||||
return d
|
||||
|
||||
def stop(self):
|
||||
return defer.succeed(True)
|
||||
async def start(self):
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
class FakeFileManager(FakeComponent):
|
||||
@@ -452,15 +439,6 @@ class FakeFileManager(FakeComponent):
|
||||
def component(self):
|
||||
return mock.Mock(spec=EncryptedFileManager)
|
||||
|
||||
def start(self):
|
||||
return defer.succeed(True)
|
||||
|
||||
def stop(self):
|
||||
pass
|
||||
|
||||
def get_status(self):
|
||||
return {}
|
||||
|
||||
|
||||
create_stream_sd_file = {
|
||||
'stream_name': '746573745f66696c65',
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from twisted.internet.task import Clock
|
||||
from twisted.trial import unittest
|
||||
import asyncio
|
||||
from unittest import TestCase
|
||||
from torba.testcase import AdvanceTimeTestCase
|
||||
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT
|
||||
@@ -10,7 +11,7 @@ from lbrynet.extras.daemon import Components
|
||||
from tests import mocks
|
||||
|
||||
|
||||
class TestComponentManager(unittest.TestCase):
|
||||
class TestComponentManager(TestCase):
|
||||
def setUp(self):
|
||||
mocks.mock_conf_settings(self)
|
||||
|
||||
@@ -60,7 +61,7 @@ class TestComponentManager(unittest.TestCase):
|
||||
self.component_manager.get_component("random_component")
|
||||
|
||||
|
||||
class TestComponentManagerOverrides(unittest.TestCase):
|
||||
class TestComponentManagerOverrides(TestCase):
|
||||
def setUp(self):
|
||||
mocks.mock_conf_settings(self)
|
||||
|
||||
@@ -91,55 +92,57 @@ class TestComponentManagerOverrides(unittest.TestCase):
|
||||
ComponentManager(randomComponent=FakeRandomComponent)
|
||||
|
||||
|
||||
class TestComponentManagerProperStart(unittest.TestCase):
|
||||
class TestComponentManagerProperStart(AdvanceTimeTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.reactor = Clock()
|
||||
mocks.mock_conf_settings(self)
|
||||
self.component_manager = ComponentManager(
|
||||
skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT,
|
||||
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT,
|
||||
HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT, RATE_LIMITER_COMPONENT,
|
||||
EXCHANGE_RATE_MANAGER_COMPONENT],
|
||||
reactor=self.reactor,
|
||||
wallet=mocks.FakeDelayedWallet,
|
||||
file_manager=mocks.FakeDelayedFileManager,
|
||||
blob_manager=mocks.FakeDelayedBlobManager
|
||||
)
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
async def test_proper_starting_of_components(self):
|
||||
asyncio.create_task(self.component_manager.setup())
|
||||
|
||||
def test_proper_starting_of_components(self):
|
||||
self.component_manager.setup()
|
||||
await self.advance(0)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
self.assertFalse(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('file_manager').running)
|
||||
|
||||
self.reactor.advance(1)
|
||||
await self.advance(1)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
self.assertTrue(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('file_manager').running)
|
||||
|
||||
self.reactor.advance(1)
|
||||
await self.advance(1)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
self.assertTrue(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('file_manager').running)
|
||||
|
||||
def test_proper_stopping_of_components(self):
|
||||
self.component_manager.setup()
|
||||
self.reactor.advance(1)
|
||||
self.reactor.advance(1)
|
||||
self.component_manager.stop()
|
||||
async def test_proper_stopping_of_components(self):
|
||||
asyncio.create_task(self.component_manager.setup())
|
||||
await self.advance(0)
|
||||
await self.advance(1)
|
||||
await self.advance(1)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
self.assertTrue(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('file_manager').running)
|
||||
|
||||
asyncio.create_task(self.component_manager.stop())
|
||||
await self.advance(0)
|
||||
self.assertFalse(self.component_manager.get_component('file_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
|
||||
self.reactor.advance(1)
|
||||
await self.advance(1)
|
||||
self.assertFalse(self.component_manager.get_component('file_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertTrue(self.component_manager.get_component('wallet').running)
|
||||
|
||||
self.reactor.advance(1)
|
||||
await self.advance(1)
|
||||
self.assertFalse(self.component_manager.get_component('file_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('blob_manager').running)
|
||||
self.assertFalse(self.component_manager.get_component('wallet').running)
|
||||
@@ -6,7 +6,6 @@ from twisted.test import proto_helpers
|
||||
from twisted.trial import unittest
|
||||
|
||||
from lbrynet.p2p import Peer
|
||||
from lbrynet.p2p.server import BlobRequestHandler
|
||||
from lbrynet.p2p.PaymentRateManager import NegotiatedPaymentRateManager, BasePaymentRateManager
|
||||
from tests.mocks import BlobAvailabilityTracker as DummyBlobAvailabilityTracker, mock_conf_settings
|
||||
|
||||
@@ -17,39 +16,37 @@ class TestBlobRequestHandlerQueries(unittest.TestCase):
|
||||
self.blob_manager = mock.Mock()
|
||||
self.payment_rate_manager = NegotiatedPaymentRateManager(
|
||||
BasePaymentRateManager(0.001), DummyBlobAvailabilityTracker())
|
||||
from lbrynet.p2p.server import BlobRequestHandler
|
||||
self.handler = BlobRequestHandler.BlobRequestHandler(
|
||||
self.blob_manager, None, self.payment_rate_manager, None)
|
||||
|
||||
def test_empty_response_when_empty_query(self):
|
||||
self.assertEqual({}, self.successResultOf(self.handler.handle_queries({})))
|
||||
self.assertEqual({}, self.handler.handle_queries({}))
|
||||
|
||||
def test_error_set_when_rate_is_missing(self):
|
||||
query = {'requested_blob': 'blob'}
|
||||
deferred = self.handler.handle_queries(query)
|
||||
response = {'incoming_blob': {'error': 'RATE_UNSET'}}
|
||||
self.assertEqual(response, self.successResultOf(deferred))
|
||||
self.assertEqual(response, self.handler.handle_queries(query))
|
||||
|
||||
def test_error_set_when_rate_too_low(self):
|
||||
query = {
|
||||
'blob_data_payment_rate': -1.0,
|
||||
'requested_blob': 'blob'
|
||||
}
|
||||
deferred = self.handler.handle_queries(query)
|
||||
response = {
|
||||
'blob_data_payment_rate': 'RATE_TOO_LOW',
|
||||
'incoming_blob': {'error': 'RATE_UNSET'}
|
||||
}
|
||||
self.assertEqual(response, self.successResultOf(deferred))
|
||||
self.assertEqual(response, self.handler.handle_queries(query))
|
||||
|
||||
def test_response_when_rate_too_low(self):
|
||||
query = {
|
||||
'blob_data_payment_rate': -1.0,
|
||||
}
|
||||
deferred = self.handler.handle_queries(query)
|
||||
response = {
|
||||
'blob_data_payment_rate': 'RATE_TOO_LOW',
|
||||
}
|
||||
self.assertEqual(response, self.successResultOf(deferred))
|
||||
self.assertEqual(response, self.handler.handle_queries(query))
|
||||
|
||||
def test_blob_unavailable_when_blob_not_validated(self):
|
||||
blob = mock.Mock()
|
||||
@@ -59,12 +56,11 @@ class TestBlobRequestHandlerQueries(unittest.TestCase):
|
||||
'blob_data_payment_rate': 1.0,
|
||||
'requested_blob': 'blob'
|
||||
}
|
||||
deferred = self.handler.handle_queries(query)
|
||||
response = {
|
||||
'blob_data_payment_rate': 'RATE_ACCEPTED',
|
||||
'incoming_blob': {'error': 'BLOB_UNAVAILABLE'}
|
||||
}
|
||||
self.assertEqual(response, self.successResultOf(deferred))
|
||||
self.assertEqual(response, self.handler.handle_queries(query))
|
||||
|
||||
def test_blob_unavailable_when_blob_cannot_be_opened(self):
|
||||
blob = mock.Mock()
|
||||
@@ -75,12 +71,11 @@ class TestBlobRequestHandlerQueries(unittest.TestCase):
|
||||
'blob_data_payment_rate': 0.0,
|
||||
'requested_blob': 'blob'
|
||||
}
|
||||
deferred = self.handler.handle_queries(query)
|
||||
response = {
|
||||
'blob_data_payment_rate': 'RATE_ACCEPTED',
|
||||
'incoming_blob': {'error': 'BLOB_UNAVAILABLE'}
|
||||
}
|
||||
self.assertEqual(response, self.successResultOf(deferred))
|
||||
self.assertEqual(response, self.handler.handle_queries(query))
|
||||
|
||||
def test_blob_details_are_set_when_all_conditions_are_met(self):
|
||||
blob = mock.Mock()
|
||||
@@ -96,7 +91,6 @@ class TestBlobRequestHandlerQueries(unittest.TestCase):
|
||||
'blob_data_payment_rate': 1.0,
|
||||
'requested_blob': 'blob'
|
||||
}
|
||||
deferred = self.handler.handle_queries(query)
|
||||
response = {
|
||||
'blob_data_payment_rate': 'RATE_ACCEPTED',
|
||||
'incoming_blob': {
|
||||
@@ -104,12 +98,12 @@ class TestBlobRequestHandlerQueries(unittest.TestCase):
|
||||
'length': 42
|
||||
}
|
||||
}
|
||||
result = self.successResultOf(deferred)
|
||||
self.assertEqual(response, result)
|
||||
self.assertEqual(response, self.handler.handle_queries(query))
|
||||
|
||||
|
||||
class TestBlobRequestHandlerSender(unittest.TestCase):
|
||||
def test_nothing_happens_if_not_currently_uploading(self):
|
||||
from lbrynet.p2p.server import BlobRequestHandler
|
||||
handler = BlobRequestHandler.BlobRequestHandler(None, None, None, None)
|
||||
handler.currently_uploading = None
|
||||
deferred = handler.send_blob_if_requested(None)
|
||||
@@ -119,6 +113,7 @@ class TestBlobRequestHandlerSender(unittest.TestCase):
|
||||
# TODO: also check that the expected payment values are set
|
||||
consumer = proto_helpers.StringTransport()
|
||||
test_file = BytesIO(b'test')
|
||||
from lbrynet.p2p.server import BlobRequestHandler
|
||||
handler = BlobRequestHandler.BlobRequestHandler(None, None, None, None)
|
||||
handler.peer = mock.create_autospec(Peer.Peer)
|
||||
handler.currently_uploading = mock.Mock()
|
||||
|
||||
@@ -8,6 +8,7 @@ from twisted.internet import defer
|
||||
|
||||
from tests.test_utils import random_lbry_hash
|
||||
from lbrynet.p2p.BlobManager import DiskBlobManager
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.p2p.Peer import Peer
|
||||
from lbrynet import conf
|
||||
@@ -21,14 +22,13 @@ class BlobManagerTest(unittest.TestCase):
|
||||
conf.initialize_settings(False)
|
||||
self.blob_dir = tempfile.mkdtemp()
|
||||
self.db_dir = tempfile.mkdtemp()
|
||||
self.bm = DiskBlobManager(self.blob_dir, SQLiteStorage(self.db_dir))
|
||||
self.bm = DiskBlobManager(self.blob_dir, SQLiteStorage(':memory:'))
|
||||
self.peer = Peer('somehost', 22)
|
||||
yield self.bm.storage.setup()
|
||||
yield f2d(self.bm.storage.open())
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def tearDown(self):
|
||||
yield self.bm.stop()
|
||||
yield self.bm.storage.stop()
|
||||
yield f2d(self.bm.storage.close())
|
||||
shutil.rmtree(self.blob_dir)
|
||||
shutil.rmtree(self.db_dir)
|
||||
|
||||
@@ -44,10 +44,10 @@ class BlobManagerTest(unittest.TestCase):
|
||||
blob_hash = out
|
||||
|
||||
# create new blob
|
||||
yield self.bm.setup()
|
||||
blob = yield self.bm.get_blob(blob_hash, len(data))
|
||||
yield f2d(self.bm.setup())
|
||||
blob = self.bm.get_blob(blob_hash, len(data))
|
||||
|
||||
writer, finished_d = yield blob.open_for_writing(self.peer)
|
||||
writer, finished_d = blob.open_for_writing(self.peer)
|
||||
yield writer.write(data)
|
||||
yield self.bm.blob_completed(blob, should_announce)
|
||||
|
||||
@@ -80,7 +80,7 @@ class BlobManagerTest(unittest.TestCase):
|
||||
self.assertFalse(os.path.isfile(os.path.join(self.blob_dir, blob_hash)))
|
||||
blobs = yield self.bm.get_all_verified_blobs()
|
||||
self.assertEqual(len(blobs), 0)
|
||||
blobs = yield self.bm.storage.get_all_blob_hashes()
|
||||
blobs = yield f2d(self.bm.storage.get_all_blob_hashes())
|
||||
self.assertEqual(len(blobs), 0)
|
||||
self.assertNotIn(blob_hash, self.bm.blobs)
|
||||
|
||||
@@ -97,6 +97,7 @@ class BlobManagerTest(unittest.TestCase):
|
||||
# Test that a blob that is opened for writing will not be deleted
|
||||
|
||||
# create blobs
|
||||
blob_hash = None
|
||||
blob_hashes = []
|
||||
for i in range(0, 10):
|
||||
blob_hash = yield self._create_and_add_blob()
|
||||
@@ -105,11 +106,11 @@ class BlobManagerTest(unittest.TestCase):
|
||||
self.assertEqual(len(blobs), 10)
|
||||
|
||||
# open the last blob
|
||||
blob = yield self.bm.get_blob(blob_hashes[-1])
|
||||
w, finished_d = yield blob.open_for_writing(self.peer)
|
||||
blob = self.bm.get_blob(blob_hashes[-1])
|
||||
w, finished_d = blob.open_for_writing(self.peer)
|
||||
|
||||
# schedule a close, just to leave the reactor clean
|
||||
finished_d.addBoth(lambda x:None)
|
||||
finished_d.addBoth(lambda x: None)
|
||||
self.addCleanup(w.close)
|
||||
|
||||
# delete the last blob and check if it still exists
|
||||
|
||||
@@ -16,7 +16,7 @@ class HTTPBlobDownloaderTest(unittest.TestCase):
|
||||
self.blob_hash = ('d17272b17a1ad61c4316ac13a651c2b0952063214a81333e'
|
||||
'838364b01b2f07edbd165bb7ec60d2fb2f337a2c02923852')
|
||||
self.blob = BlobFile(self.blob_dir, self.blob_hash)
|
||||
self.blob_manager.get_blob.side_effect = lambda _: defer.succeed(self.blob)
|
||||
self.blob_manager.get_blob.side_effect = lambda _: self.blob
|
||||
self.response = MagicMock(code=200, length=400)
|
||||
self.client.get.side_effect = lambda uri: defer.succeed(self.response)
|
||||
self.downloader = HTTPBlobDownloader(self.blob_manager, [self.blob_hash], ['server1'], self.client, retry=False)
|
||||
|
||||
@@ -62,20 +62,21 @@ class BlobFileTest(unittest.TestCase):
|
||||
blob_file = BlobFile(self.blob_dir, self.fake_content_hash)
|
||||
self.assertFalse(blob_file.verified)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_delete_fail(self):
|
||||
# deletes should fail if being written to
|
||||
blob_file = BlobFile(self.blob_dir, self.fake_content_hash, self.fake_content_len)
|
||||
writer, finished_d = blob_file.open_for_writing(peer=1)
|
||||
yield self.assertFailure(blob_file.delete(), ValueError)
|
||||
with self.assertRaises(ValueError):
|
||||
blob_file.delete()
|
||||
writer.write(self.fake_content)
|
||||
writer.close()
|
||||
|
||||
# deletes should fail if being read and not closed
|
||||
blob_file = BlobFile(self.blob_dir, self.fake_content_hash, self.fake_content_len)
|
||||
self.assertTrue(blob_file.verified)
|
||||
f = blob_file.open_for_reading()
|
||||
yield self.assertFailure(blob_file.delete(), ValueError)
|
||||
r = blob_file.open_for_reading() # must be set to variable otherwise it gets garbage collected
|
||||
with self.assertRaises(ValueError):
|
||||
blob_file.delete()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_too_much_write(self):
|
||||
|
||||
@@ -6,6 +6,7 @@ from copy import deepcopy
|
||||
from twisted.internet import defer
|
||||
from twisted.trial import unittest
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage, open_file_for_writing
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from tests.test_utils import random_lbry_hash
|
||||
@@ -85,18 +86,18 @@ class StorageTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
conf.initialize_settings(False)
|
||||
self.db_dir = tempfile.mkdtemp()
|
||||
self.storage = SQLiteStorage(self.db_dir)
|
||||
yield self.storage.setup()
|
||||
self.storage = SQLiteStorage(':memory:')
|
||||
yield f2d(self.storage.open())
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def tearDown(self):
|
||||
yield self.storage.stop()
|
||||
yield f2d(self.storage.close())
|
||||
shutil.rmtree(self.db_dir)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def store_fake_blob(self, blob_hash, blob_length=100, next_announce=0, should_announce=0):
|
||||
yield self.storage.add_completed_blob(blob_hash, blob_length, next_announce,
|
||||
should_announce, "finished")
|
||||
yield f2d(self.storage.add_completed_blob(blob_hash, blob_length, next_announce,
|
||||
should_announce, "finished"))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def store_fake_stream_blob(self, stream_hash, blob_hash, blob_num, length=100, iv="DEADBEEF"):
|
||||
@@ -105,13 +106,13 @@ class StorageTest(unittest.TestCase):
|
||||
}
|
||||
if length:
|
||||
blob_info['length'] = length
|
||||
yield self.storage.add_blobs_to_stream(stream_hash, [blob_info])
|
||||
yield f2d(self.storage.add_blobs_to_stream(stream_hash, [blob_info]))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def store_fake_stream(self, stream_hash, sd_hash, file_name="fake_file", key="DEADBEEF",
|
||||
blobs=[]):
|
||||
yield self.storage.store_stream(stream_hash, sd_hash, file_name, key,
|
||||
file_name, blobs)
|
||||
yield f2d(self.storage.store_stream(stream_hash, sd_hash, file_name, key,
|
||||
file_name, blobs))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def make_and_store_fake_stream(self, blob_count=2, stream_hash=None, sd_hash=None):
|
||||
@@ -135,9 +136,9 @@ class StorageTest(unittest.TestCase):
|
||||
class TestSetup(StorageTest):
|
||||
@defer.inlineCallbacks
|
||||
def test_setup(self):
|
||||
files = yield self.storage.get_all_lbry_files()
|
||||
files = yield f2d(self.storage.get_all_lbry_files())
|
||||
self.assertEqual(len(files), 0)
|
||||
blobs = yield self.storage.get_all_blob_hashes()
|
||||
blobs = yield f2d(self.storage.get_all_blob_hashes())
|
||||
self.assertEqual(len(blobs), 0)
|
||||
|
||||
|
||||
@@ -146,17 +147,17 @@ class BlobStorageTests(StorageTest):
|
||||
def test_store_blob(self):
|
||||
blob_hash = random_lbry_hash()
|
||||
yield self.store_fake_blob(blob_hash)
|
||||
blob_hashes = yield self.storage.get_all_blob_hashes()
|
||||
blob_hashes = yield f2d(self.storage.get_all_blob_hashes())
|
||||
self.assertEqual(blob_hashes, [blob_hash])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_delete_blob(self):
|
||||
blob_hash = random_lbry_hash()
|
||||
yield self.store_fake_blob(blob_hash)
|
||||
blob_hashes = yield self.storage.get_all_blob_hashes()
|
||||
blob_hashes = yield f2d(self.storage.get_all_blob_hashes())
|
||||
self.assertEqual(blob_hashes, [blob_hash])
|
||||
yield self.storage.delete_blobs_from_db(blob_hashes)
|
||||
blob_hashes = yield self.storage.get_all_blob_hashes()
|
||||
yield f2d(self.storage.delete_blobs_from_db(blob_hashes))
|
||||
blob_hashes = yield f2d(self.storage.get_all_blob_hashes())
|
||||
self.assertEqual(blob_hashes, [])
|
||||
|
||||
|
||||
@@ -172,13 +173,13 @@ class SupportsStorageTests(StorageTest):
|
||||
} for i in range(20)]
|
||||
expected_supports = {}
|
||||
for idx, claim_id in enumerate(claim_ids):
|
||||
yield self.storage.save_supports(claim_id, random_supports[idx*2:idx*2+2])
|
||||
yield f2d(self.storage.save_supports(claim_id, random_supports[idx*2:idx*2+2]))
|
||||
for random_support in random_supports[idx*2:idx*2+2]:
|
||||
random_support['claim_id'] = claim_id
|
||||
expected_supports.setdefault(claim_id, []).append(random_support)
|
||||
supports = yield self.storage.get_supports(claim_ids[0])
|
||||
supports = yield f2d(self.storage.get_supports(claim_ids[0]))
|
||||
self.assertEqual(supports, expected_supports[claim_ids[0]])
|
||||
all_supports = yield self.storage.get_supports(*claim_ids)
|
||||
all_supports = yield f2d(self.storage.get_supports(*claim_ids))
|
||||
for support in all_supports:
|
||||
self.assertIn(support, expected_supports[support['claim_id']])
|
||||
|
||||
@@ -199,55 +200,56 @@ class StreamStorageTests(StorageTest):
|
||||
yield self.store_fake_stream_blob(stream_hash, blob1, 1)
|
||||
yield self.store_fake_stream_blob(stream_hash, blob2, 2)
|
||||
|
||||
stream_blobs = yield self.storage.get_blobs_for_stream(stream_hash)
|
||||
stream_blobs = yield f2d(self.storage.get_blobs_for_stream(stream_hash))
|
||||
stream_blob_hashes = [b.blob_hash for b in stream_blobs]
|
||||
self.assertListEqual(stream_blob_hashes, [blob1, blob2])
|
||||
|
||||
blob_hashes = yield self.storage.get_all_blob_hashes()
|
||||
blob_hashes = yield f2d(self.storage.get_all_blob_hashes())
|
||||
self.assertSetEqual(set(blob_hashes), {sd_hash, blob1, blob2})
|
||||
|
||||
stream_blobs = yield self.storage.get_blobs_for_stream(stream_hash)
|
||||
stream_blobs = yield f2d(self.storage.get_blobs_for_stream(stream_hash))
|
||||
stream_blob_hashes = [b.blob_hash for b in stream_blobs]
|
||||
self.assertListEqual(stream_blob_hashes, [blob1, blob2])
|
||||
|
||||
yield self.storage.set_should_announce(sd_hash, 1, 1)
|
||||
yield self.storage.set_should_announce(blob1, 1, 1)
|
||||
yield f2d(self.storage.set_should_announce(sd_hash, 1, 1))
|
||||
yield f2d(self.storage.set_should_announce(blob1, 1, 1))
|
||||
|
||||
should_announce_count = yield self.storage.count_should_announce_blobs()
|
||||
should_announce_count = yield f2d(self.storage.count_should_announce_blobs())
|
||||
self.assertEqual(should_announce_count, 2)
|
||||
should_announce_hashes = yield self.storage.get_blobs_to_announce()
|
||||
should_announce_hashes = yield f2d(self.storage.get_blobs_to_announce())
|
||||
self.assertSetEqual(set(should_announce_hashes), {sd_hash, blob1})
|
||||
|
||||
stream_hashes = yield self.storage.get_all_streams()
|
||||
stream_hashes = yield f2d(self.storage.get_all_streams())
|
||||
self.assertListEqual(stream_hashes, [stream_hash])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_delete_stream(self):
|
||||
stream_hash = random_lbry_hash()
|
||||
yield self.test_store_stream(stream_hash)
|
||||
yield self.storage.delete_stream(stream_hash)
|
||||
stream_hashes = yield self.storage.get_all_streams()
|
||||
yield f2d(self.storage.delete_stream(stream_hash))
|
||||
stream_hashes = yield f2d(self.storage.get_all_streams())
|
||||
self.assertListEqual(stream_hashes, [])
|
||||
|
||||
stream_blobs = yield self.storage.get_blobs_for_stream(stream_hash)
|
||||
stream_blobs = yield f2d(self.storage.get_blobs_for_stream(stream_hash))
|
||||
self.assertListEqual(stream_blobs, [])
|
||||
blob_hashes = yield self.storage.get_all_blob_hashes()
|
||||
blob_hashes = yield f2d(self.storage.get_all_blob_hashes())
|
||||
self.assertListEqual(blob_hashes, [])
|
||||
|
||||
|
||||
class FileStorageTests(StorageTest):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_setup_output(self):
|
||||
file_name = 'encrypted_file_saver_test.tmp'
|
||||
self.assertFalse(os.path.isfile(file_name))
|
||||
written_to = yield open_file_for_writing(self.db_dir, file_name)
|
||||
written_to = yield f2d(open_file_for_writing(self.db_dir, file_name))
|
||||
self.assertEqual(written_to, file_name)
|
||||
self.assertTrue(os.path.isfile(os.path.join(self.db_dir, file_name)))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_store_file(self):
|
||||
download_directory = self.db_dir
|
||||
out = yield self.storage.get_all_lbry_files()
|
||||
out = yield f2d(self.storage.get_all_lbry_files())
|
||||
self.assertEqual(len(out), 0)
|
||||
|
||||
stream_hash = random_lbry_hash()
|
||||
@@ -265,29 +267,30 @@ class FileStorageTests(StorageTest):
|
||||
|
||||
blob_data_rate = 0
|
||||
file_name = "test file"
|
||||
out = yield self.storage.save_published_file(
|
||||
out = yield f2d(self.storage.save_published_file(
|
||||
stream_hash, file_name, download_directory, blob_data_rate
|
||||
)
|
||||
rowid = yield self.storage.get_rowid_for_stream_hash(stream_hash)
|
||||
))
|
||||
rowid = yield f2d(self.storage.get_rowid_for_stream_hash(stream_hash))
|
||||
self.assertEqual(out, rowid)
|
||||
|
||||
files = yield self.storage.get_all_lbry_files()
|
||||
files = yield f2d(self.storage.get_all_lbry_files())
|
||||
self.assertEqual(1, len(files))
|
||||
|
||||
status = yield self.storage.get_lbry_file_status(rowid)
|
||||
status = yield f2d(self.storage.get_lbry_file_status(rowid))
|
||||
self.assertEqual(status, ManagedEncryptedFileDownloader.STATUS_STOPPED)
|
||||
|
||||
running = ManagedEncryptedFileDownloader.STATUS_RUNNING
|
||||
yield self.storage.change_file_status(rowid, running)
|
||||
status = yield self.storage.get_lbry_file_status(rowid)
|
||||
yield f2d(self.storage.change_file_status(rowid, running))
|
||||
status = yield f2d(self.storage.get_lbry_file_status(rowid))
|
||||
self.assertEqual(status, ManagedEncryptedFileDownloader.STATUS_RUNNING)
|
||||
|
||||
|
||||
class ContentClaimStorageTests(StorageTest):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_store_content_claim(self):
|
||||
download_directory = self.db_dir
|
||||
out = yield self.storage.get_all_lbry_files()
|
||||
out = yield f2d(self.storage.get_all_lbry_files())
|
||||
self.assertEqual(len(out), 0)
|
||||
|
||||
stream_hash = random_lbry_hash()
|
||||
@@ -300,32 +303,32 @@ class ContentClaimStorageTests(StorageTest):
|
||||
yield self.make_and_store_fake_stream(blob_count=2, stream_hash=stream_hash, sd_hash=sd_hash)
|
||||
blob_data_rate = 0
|
||||
file_name = "test file"
|
||||
yield self.storage.save_published_file(
|
||||
yield f2d(self.storage.save_published_file(
|
||||
stream_hash, file_name, download_directory, blob_data_rate
|
||||
)
|
||||
yield self.storage.save_claims([fake_claim_info])
|
||||
yield self.storage.save_content_claim(stream_hash, fake_outpoint)
|
||||
stored_content_claim = yield self.storage.get_content_claim(stream_hash)
|
||||
))
|
||||
yield f2d(self.storage.save_claims([fake_claim_info]))
|
||||
yield f2d(self.storage.save_content_claim(stream_hash, fake_outpoint))
|
||||
stored_content_claim = yield f2d(self.storage.get_content_claim(stream_hash))
|
||||
self.assertDictEqual(stored_content_claim, fake_claim_info)
|
||||
|
||||
stream_hashes = yield self.storage.get_old_stream_hashes_for_claim_id(fake_claim_info['claim_id'],
|
||||
stream_hash)
|
||||
stream_hashes = yield f2d(self.storage.get_old_stream_hashes_for_claim_id(fake_claim_info['claim_id'],
|
||||
stream_hash))
|
||||
self.assertListEqual(stream_hashes, [])
|
||||
|
||||
# test that we can't associate a claim update with a new stream to the file
|
||||
second_stream_hash, second_sd_hash = random_lbry_hash(), random_lbry_hash()
|
||||
yield self.make_and_store_fake_stream(blob_count=2, stream_hash=second_stream_hash, sd_hash=second_sd_hash)
|
||||
with self.assertRaisesRegex(Exception, "stream mismatch"):
|
||||
yield self.storage.save_content_claim(second_stream_hash, fake_outpoint)
|
||||
yield f2d(self.storage.save_content_claim(second_stream_hash, fake_outpoint))
|
||||
|
||||
# test that we can associate a new claim update containing the same stream to the file
|
||||
update_info = deepcopy(fake_claim_info)
|
||||
update_info['txid'] = "beef0000" * 12
|
||||
update_info['nout'] = 0
|
||||
second_outpoint = "%s:%i" % (update_info['txid'], update_info['nout'])
|
||||
yield self.storage.save_claims([update_info])
|
||||
yield self.storage.save_content_claim(stream_hash, second_outpoint)
|
||||
update_info_result = yield self.storage.get_content_claim(stream_hash)
|
||||
yield f2d(self.storage.save_claims([update_info]))
|
||||
yield f2d(self.storage.save_content_claim(stream_hash, second_outpoint))
|
||||
update_info_result = yield f2d(self.storage.get_content_claim(stream_hash))
|
||||
self.assertDictEqual(update_info_result, update_info)
|
||||
|
||||
# test that we can't associate an update with a mismatching claim id
|
||||
@@ -337,8 +340,8 @@ class ContentClaimStorageTests(StorageTest):
|
||||
with self.assertRaisesRegex(Exception, "mismatching claim ids when updating stream "
|
||||
"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef "
|
||||
"vs beef0002beef0002beef0002beef0002beef0002"):
|
||||
yield self.storage.save_claims([invalid_update_info])
|
||||
yield self.storage.save_content_claim(stream_hash, invalid_update_outpoint)
|
||||
current_claim_info = yield self.storage.get_content_claim(stream_hash)
|
||||
yield f2d(self.storage.save_claims([invalid_update_info]))
|
||||
yield f2d(self.storage.save_content_claim(stream_hash, invalid_update_outpoint))
|
||||
current_claim_info = yield f2d(self.storage.get_content_claim(stream_hash))
|
||||
# this should still be the previous update
|
||||
self.assertDictEqual(current_claim_info, update_info)
|
||||
@@ -3,6 +3,7 @@ from twisted.trial import unittest
|
||||
from twisted.internet import defer
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers.algorithms import AES
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.extras.daemon.PeerManager import PeerManager
|
||||
from lbrynet.p2p.StreamDescriptor import get_sd_info, BlobStreamDescriptorReader
|
||||
from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier
|
||||
@@ -44,20 +45,20 @@ class CreateEncryptedFileTest(unittest.TestCase):
|
||||
self.peer_finder = FakePeerFinder(5553, self.peer_manager, 2)
|
||||
self.rate_limiter = DummyRateLimiter()
|
||||
self.sd_identifier = StreamDescriptorIdentifier()
|
||||
self.storage = SQLiteStorage(self.tmp_db_dir)
|
||||
self.storage = SQLiteStorage(':memory:')
|
||||
self.blob_manager = DiskBlobManager(self.tmp_blob_dir, self.storage)
|
||||
self.prm = OnlyFreePaymentsManager()
|
||||
self.lbry_file_manager = EncryptedFileManager(self.peer_finder, self.rate_limiter, self.blob_manager,
|
||||
self.wallet, self.prm, self.storage, self.sd_identifier)
|
||||
d = self.storage.setup()
|
||||
d.addCallback(lambda _: self.lbry_file_manager.setup())
|
||||
d = f2d(self.storage.open())
|
||||
d.addCallback(lambda _: f2d(self.lbry_file_manager.setup()))
|
||||
return d
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def tearDown(self):
|
||||
yield self.lbry_file_manager.stop()
|
||||
yield self.blob_manager.stop()
|
||||
yield self.storage.stop()
|
||||
yield f2d(self.blob_manager.stop())
|
||||
yield f2d(self.storage.close())
|
||||
rm_db_and_blob_dir(self.tmp_db_dir, self.tmp_blob_dir)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -77,7 +78,7 @@ class CreateEncryptedFileTest(unittest.TestCase):
|
||||
"3e62e81a2e8945b0db7c94f1852e70e371d917b994352c"
|
||||
filename = 'test.file'
|
||||
lbry_file = yield self.create_file(filename)
|
||||
sd_hash = yield self.storage.get_sd_blob_hash_for_stream(lbry_file.stream_hash)
|
||||
sd_hash = yield f2d(self.storage.get_sd_blob_hash_for_stream(lbry_file.stream_hash))
|
||||
|
||||
# read the sd blob file
|
||||
sd_blob = self.blob_manager.blobs[sd_hash]
|
||||
@@ -85,7 +86,7 @@ class CreateEncryptedFileTest(unittest.TestCase):
|
||||
sd_file_info = yield sd_reader.get_info()
|
||||
|
||||
# this comes from the database, the blobs returned are sorted
|
||||
sd_info = yield get_sd_info(self.storage, lbry_file.stream_hash, include_blobs=True)
|
||||
sd_info = yield f2d(get_sd_info(self.storage, lbry_file.stream_hash, include_blobs=True))
|
||||
self.maxDiff = None
|
||||
unicode_sd_info = json.loads(json.dumps(sd_info, sort_keys=True, cls=JSONBytesEncoder))
|
||||
self.assertDictEqual(unicode_sd_info, sd_file_info)
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
from unittest import mock
|
||||
from unittest import mock, skip
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.trial import unittest
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.daemon.auth import server
|
||||
#from lbrynet.extras.daemon.auth import server
|
||||
from tests.mocks import mock_conf_settings
|
||||
|
||||
|
||||
@skip
|
||||
class AuthJSONRPCServerTest(unittest.TestCase):
|
||||
# TODO: move to using a base class for tests
|
||||
# and add useful general utilities like this
|
||||
|
||||
@@ -5,17 +5,18 @@ from twisted.internet import defer
|
||||
from twisted.trial import unittest
|
||||
|
||||
from lbrynet import conf
|
||||
from lbrynet.extras.compat import f2d
|
||||
from lbrynet.schema.decode import smart_decode
|
||||
from lbrynet.extras.daemon.storage import SQLiteStorage
|
||||
from lbrynet.extras.daemon.ComponentManager import ComponentManager
|
||||
from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT
|
||||
from lbrynet.extras.daemon.Components import f2d
|
||||
from lbrynet.extras.daemon.Components import HASH_ANNOUNCER_COMPONENT, REFLECTOR_COMPONENT
|
||||
from lbrynet.extras.daemon.Components import UPNP_COMPONENT, BLOB_COMPONENT
|
||||
from lbrynet.extras.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
|
||||
from lbrynet.extras.daemon.Components import RATE_LIMITER_COMPONENT, HEADERS_COMPONENT, FILE_MANAGER_COMPONENT
|
||||
from lbrynet.extras.daemon.Daemon import Daemon as LBRYDaemon
|
||||
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
|
||||
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
|
||||
from lbrynet.extras.wallet import LbryWalletManager
|
||||
from torba.client.wallet import Wallet
|
||||
|
||||
@@ -130,14 +131,14 @@ class TestJsonRpc(unittest.TestCase):
|
||||
self.test_daemon.wallet_manager.is_first_run = False
|
||||
self.test_daemon.wallet_manager.get_best_blockhash = noop
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_status(self):
|
||||
d = defer.maybeDeferred(self.test_daemon.jsonrpc_status)
|
||||
d.addCallback(lambda status: self.assertDictContainsSubset({'is_running': False}, status))
|
||||
status = yield f2d(self.test_daemon.jsonrpc_status())
|
||||
self.assertDictContainsSubset({'is_running': False}, status)
|
||||
|
||||
def test_help(self):
|
||||
d = defer.maybeDeferred(self.test_daemon.jsonrpc_help, command='status')
|
||||
d.addCallback(lambda result: self.assertSubstring('daemon status', result['help']))
|
||||
# self.assertSubstring('daemon status', d.result)
|
||||
result = self.test_daemon.jsonrpc_help(command='status')
|
||||
self.assertSubstring('daemon status', result['help'])
|
||||
|
||||
if is_android():
|
||||
test_help.skip = "Test cannot pass on Android because PYTHONOPTIMIZE removes the docstrings."
|
||||
@@ -162,42 +163,42 @@ class TestFileListSorting(unittest.TestCase):
|
||||
'ashlee27', 'bfrederick', 'brittanyhicks', 'davidsonjeffrey', 'heidiherring',
|
||||
'jlewis', 'kswanson', 'michelle50', 'richard64', 'xsteele'
|
||||
]
|
||||
return self.test_daemon.component_manager.setup()
|
||||
return f2d(self.test_daemon.component_manager.setup())
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_sort_by_points_paid_no_direction_specified(self):
|
||||
sort_options = ['points_paid']
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
self.assertEqual(self.test_points_paid, [f['points_paid'] for f in file_list])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_sort_by_points_paid_ascending(self):
|
||||
sort_options = ['points_paid,asc']
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
self.assertEqual(self.test_points_paid, [f['points_paid'] for f in file_list])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_sort_by_points_paid_descending(self):
|
||||
sort_options = ['points_paid, desc']
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
self.assertEqual(list(reversed(self.test_points_paid)), [f['points_paid'] for f in file_list])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_sort_by_file_name_no_direction_specified(self):
|
||||
sort_options = ['file_name']
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
self.assertEqual(self.test_file_names, [f['file_name'] for f in file_list])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_sort_by_file_name_ascending(self):
|
||||
sort_options = ['file_name,\nasc']
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
self.assertEqual(self.test_file_names, [f['file_name'] for f in file_list])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_sort_by_file_name_descending(self):
|
||||
sort_options = ['\tfile_name,\n\tdesc']
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
self.assertEqual(list(reversed(self.test_file_names)), [f['file_name'] for f in file_list])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -217,21 +218,21 @@ class TestFileListSorting(unittest.TestCase):
|
||||
format_result = lambda f: 'file_name={}, points_paid={}'.format(f['file_name'], f['points_paid'])
|
||||
|
||||
sort_options = ['file_name,asc', 'points_paid,desc']
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
self.assertEqual(expected, [format_result(r) for r in file_list])
|
||||
|
||||
# Check that the list is not sorted as expected when sorted only by file_name.
|
||||
sort_options = ['file_name,asc']
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
self.assertNotEqual(expected, [format_result(r) for r in file_list])
|
||||
|
||||
# Check that the list is not sorted as expected when sorted only by points_paid.
|
||||
sort_options = ['points_paid,desc']
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
self.assertNotEqual(expected, [format_result(r) for r in file_list])
|
||||
|
||||
# Check that the list is not sorted as expected when not sorted at all.
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list()
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list())
|
||||
self.assertNotEqual(expected, [format_result(r) for r in file_list])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -239,16 +240,16 @@ class TestFileListSorting(unittest.TestCase):
|
||||
extract_authors = lambda file_list: [f['metadata']['author'] for f in file_list]
|
||||
|
||||
sort_options = ['metadata.author']
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
self.assertEqual(self.test_authors, extract_authors(file_list))
|
||||
|
||||
# Check that the list matches the expected in reverse when sorting in descending order.
|
||||
sort_options = ['metadata.author,desc']
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
self.assertEqual(list(reversed(self.test_authors)), extract_authors(file_list))
|
||||
|
||||
# Check that the list is not sorted as expected when not sorted at all.
|
||||
file_list = yield self.test_daemon.jsonrpc_file_list()
|
||||
file_list = yield f2d(self.test_daemon.jsonrpc_file_list())
|
||||
self.assertNotEqual(self.test_authors, extract_authors(file_list))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
@@ -256,11 +257,11 @@ class TestFileListSorting(unittest.TestCase):
|
||||
sort_options = ['meta.author']
|
||||
expected_message = "Failed to get 'meta.author', key 'meta' was not found."
|
||||
with self.assertRaisesRegex(Exception, expected_message):
|
||||
yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
sort_options = ['metadata.foo.bar']
|
||||
expected_message = "Failed to get 'metadata.foo.bar', key 'foo' was not found."
|
||||
with self.assertRaisesRegex(Exception, expected_message):
|
||||
yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
|
||||
yield f2d(self.test_daemon.jsonrpc_file_list(sort=sort_options))
|
||||
|
||||
@staticmethod
|
||||
def _get_fake_lbry_files():
|
||||
@@ -269,6 +270,14 @@ class TestFileListSorting(unittest.TestCase):
|
||||
lbry_file = mock.Mock(spec=ManagedEncryptedFileDownloader)
|
||||
for attribute in metadata:
|
||||
setattr(lbry_file, attribute, metadata[attribute])
|
||||
async def get_total_bytes():
|
||||
return 0
|
||||
lbry_file.get_total_bytes = get_total_bytes
|
||||
async def status():
|
||||
return EncryptedFileStatusReport(
|
||||
'file_name', 1, 1, 'completed'
|
||||
)
|
||||
lbry_file.status = status
|
||||
faked_lbry_files.append(lbry_file)
|
||||
return faked_lbry_files
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import binascii
|
||||
from copy import deepcopy
|
||||
import unittest
|
||||
|
||||
from lbrynet.schema.signature import Signature, NAMED_SECP256K1
|
||||
from .test_data import example_003, example_010, example_010_serialized
|
||||
from .test_data import claim_id_1, claim_address_1, claim_address_2
|
||||
from .test_data import binary_claim, expected_binary_claim_decoded
|
||||
@@ -25,24 +26,25 @@ from lbrynet.schema.address import decode_address, encode_address
|
||||
|
||||
|
||||
parsed_uri_matches = [
|
||||
("test", URI("test"), False),
|
||||
("test#%s" % claim_id_1, URI("test", claim_id=claim_id_1), False),
|
||||
("test:1", URI("test", claim_sequence=1), False),
|
||||
("test$1", URI("test", bid_position=1), False),
|
||||
("lbry://test", URI("test"), False),
|
||||
("lbry://test#%s" % claim_id_1, URI("test", claim_id=claim_id_1), False),
|
||||
("lbry://test:1", URI("test", claim_sequence=1), False),
|
||||
("lbry://test$1", URI("test", bid_position=1), False),
|
||||
("@test", URI("@test"), True),
|
||||
("@test#%s" % claim_id_1, URI("@test", claim_id=claim_id_1), True),
|
||||
("@test:1", URI("@test", claim_sequence=1), True),
|
||||
("@test$1", URI("@test", bid_position=1), True),
|
||||
("lbry://@test1:1/fakepath", URI("@test1", claim_sequence=1, path="fakepath"), True),
|
||||
("lbry://@test1$1/fakepath", URI("@test1", bid_position=1, path="fakepath"), True),
|
||||
("lbry://@test1#abcdef/fakepath", URI("@test1", claim_id="abcdef", path="fakepath"), True),
|
||||
("@z", URI("@z"), True),
|
||||
("@yx", URI("@yx"), True),
|
||||
("@abc", URI("@abc"), True)
|
||||
("test", URI("test"), False, False, "test", None),
|
||||
("test#%s" % claim_id_1, URI("test", claim_id=claim_id_1), False, False, "test", None),
|
||||
("test:1", URI("test", claim_sequence=1), False, False, "test", None),
|
||||
("test$1", URI("test", bid_position=1), False, False, "test", None),
|
||||
("lbry://test", URI("test"), False, False, "test", None),
|
||||
("lbry://test#%s" % claim_id_1, URI("test", claim_id=claim_id_1), False, False, "test", None),
|
||||
("lbry://test:1", URI("test", claim_sequence=1), False, False, "test", None),
|
||||
("lbry://test$1", URI("test", bid_position=1), False, False, "test", None),
|
||||
("@test", URI("@test"), True, True, None, "@test"),
|
||||
("@test#%s" % claim_id_1, URI("@test", claim_id=claim_id_1), True, True, None, "@test"),
|
||||
("@test:1", URI("@test", claim_sequence=1), True, True, None, "@test"),
|
||||
("@test$1", URI("@test", bid_position=1), True, True, None, "@test"),
|
||||
("lbry://@test1:1/fakepath", URI("@test1", claim_sequence=1, path="fakepath"), True, False, "fakepath", "@test1"),
|
||||
("lbry://@test1$1/fakepath", URI("@test1", bid_position=1, path="fakepath"), True, False, "fakepath", "@test1"),
|
||||
("lbry://@test1#abcdef/fakepath", URI("@test1", claim_id="abcdef", path="fakepath"), True, False, "fakepath",
|
||||
"@test1"),
|
||||
("@z", URI("@z"), True, True, None, "@z"),
|
||||
("@yx", URI("@yx"), True, True, None, "@yx"),
|
||||
("@abc", URI("@abc"), True, True, None, "@abc")
|
||||
]
|
||||
|
||||
parsed_uri_raises = [
|
||||
@@ -89,16 +91,25 @@ class TestURIParser(UnitTest):
|
||||
self.longMessage = True
|
||||
|
||||
def test_uri_parse(self):
|
||||
for test_string, expected_uri_obj, is_channel in parsed_uri_matches:
|
||||
for test_string, expected_uri_obj, contains_channel, is_channel, claim_name, channel_name in parsed_uri_matches:
|
||||
try:
|
||||
# string -> URI
|
||||
self.assertEqual(URI.from_uri_string(test_string), expected_uri_obj, test_string)
|
||||
# URI -> dict -> URI
|
||||
self.assertEqual(URI.from_dict(expected_uri_obj.to_dict()), expected_uri_obj,
|
||||
test_string)
|
||||
# contains_channel
|
||||
self.assertEqual(URI.from_uri_string(test_string).contains_channel, contains_channel,
|
||||
test_string)
|
||||
# is_channel
|
||||
self.assertEqual(URI.from_uri_string(test_string).is_channel, is_channel,
|
||||
test_string)
|
||||
# claim_name
|
||||
self.assertEqual(URI.from_uri_string(test_string).claim_name, claim_name,
|
||||
test_string)
|
||||
# channel_name
|
||||
self.assertEqual(URI.from_uri_string(test_string).channel_name, channel_name,
|
||||
test_string)
|
||||
|
||||
# convert-to-string test only works if protocol is present in test_string
|
||||
if test_string.startswith('lbry://'):
|
||||
@@ -319,6 +330,132 @@ class TestSECP256k1Signatures(UnitTest):
|
||||
claim_address_1, cert)
|
||||
|
||||
|
||||
class TestDetachedNamedSECP256k1Signatures(UnitTest):
|
||||
def test_validate_detached_named_ecdsa_signature(self):
|
||||
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
|
||||
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
|
||||
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_2, claim_id_1,
|
||||
curve=SECP256k1, name='example', force_detached=True)
|
||||
signed_copy = ClaimDict.deserialize(signed.serialized)
|
||||
self.assertEqual(signed_copy.validate_signature(claim_address_2, cert, name='example'), True)
|
||||
|
||||
def test_validate_detached_named_ecdsa_signature_from_dict(self):
|
||||
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
|
||||
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
|
||||
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_2, claim_id_1,
|
||||
curve=SECP256k1, name='example', force_detached=True)
|
||||
self.assertEqual(
|
||||
signed.claim_dict['publisherSignature']['detached_signature'],
|
||||
binascii.hexlify(signed.serialized).decode()
|
||||
)
|
||||
signed_copy = ClaimDict.load_dict(signed.claim_dict)
|
||||
self.assertEqual(signed_copy.validate_signature(claim_address_2, cert, name='example'), True)
|
||||
|
||||
def test_validate_what_cant_be_serialized_back(self):
|
||||
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
|
||||
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
|
||||
original = ClaimDict.load_dict(example_010).serialized
|
||||
altered = original + b'\x00\x01\x02\x30\x50\x80\x99' # pretend this extra trash is from some unknown protobuf
|
||||
|
||||
# manually sign
|
||||
signer = get_signer(SECP256k1).load_pem(secp256k1_private_key)
|
||||
signature = signer.sign(
|
||||
b'example',
|
||||
decode_address(claim_address_2),
|
||||
altered,
|
||||
binascii.unhexlify(claim_id_1),
|
||||
)
|
||||
detached_sig = Signature(NAMED_SECP256K1(
|
||||
signature,
|
||||
binascii.unhexlify(claim_id_1),
|
||||
altered
|
||||
))
|
||||
|
||||
signed = detached_sig.serialized
|
||||
self.assertEqual(signed[85:], altered)
|
||||
signed_copy = ClaimDict.deserialize(signed)
|
||||
self.assertEqual(signed_copy.validate_signature(claim_address_2, cert, name='example'), True)
|
||||
self.assertEqual(signed, signed_copy.serialized)
|
||||
|
||||
def test_validate_what_cant_be_serialized_back_even_by_loading_back_from_dictionary(self):
|
||||
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
|
||||
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
|
||||
original = ClaimDict.load_dict(example_010).serialized
|
||||
altered = original + b'\x00\x01\x02\x30\x50\x80\x99' # pretend this extra trash is from some unknown protobuf
|
||||
|
||||
# manually sign
|
||||
signer = get_signer(SECP256k1).load_pem(secp256k1_private_key)
|
||||
signature = signer.sign(
|
||||
b'example',
|
||||
decode_address(claim_address_2),
|
||||
altered,
|
||||
binascii.unhexlify(claim_id_1),
|
||||
)
|
||||
detached_sig = Signature(NAMED_SECP256K1(
|
||||
signature,
|
||||
binascii.unhexlify(claim_id_1),
|
||||
altered
|
||||
))
|
||||
|
||||
signed = detached_sig.serialized
|
||||
self.assertEqual(signed[85:], altered)
|
||||
signed_copy = ClaimDict.deserialize(signed)
|
||||
signed_copy = ClaimDict.load_dict(signed_copy.claim_dict)
|
||||
self.assertEqual(signed_copy.validate_signature(claim_address_2, cert, name='example'), True)
|
||||
self.assertEqual(signed, signed_copy.serialized)
|
||||
|
||||
def test_fail_to_sign_with_no_claim_address(self):
|
||||
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
|
||||
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
|
||||
self.assertRaises(Exception, ClaimDict.load_dict(example_010).sign, secp256k1_private_key,
|
||||
None, claim_id_1, curve=SECP256k1, name='example', force_detached=True)
|
||||
|
||||
def test_fail_to_validate_with_no_claim_address(self):
|
||||
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
|
||||
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
|
||||
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_2, claim_id_1,
|
||||
curve=SECP256k1, name='example', force_detached=True)
|
||||
signed_copy = ClaimDict.load_protobuf(signed.protobuf)
|
||||
self.assertRaises(Exception, signed_copy.validate_signature, None, cert, name='example')
|
||||
|
||||
def test_fail_to_validate_with_no_name(self):
|
||||
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
|
||||
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
|
||||
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_2, claim_id_1,
|
||||
curve=SECP256k1, name='example', force_detached=True)
|
||||
signed_copy = ClaimDict.load_protobuf(signed.protobuf)
|
||||
self.assertRaises(Exception, signed_copy.validate_signature, None, cert, name=None)
|
||||
|
||||
def test_remove_signature_equals_unsigned(self):
|
||||
unsigned = ClaimDict.load_dict(example_010)
|
||||
signed = unsigned.sign(secp256k1_private_key, claim_address_1, claim_id_1,
|
||||
curve=SECP256k1, name='example', force_detached=True)
|
||||
self.assertEqual(unsigned.serialized, signed.serialized_no_signature)
|
||||
|
||||
def test_fail_to_validate_fake_ecdsa_signature(self):
|
||||
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_1, claim_id_1,
|
||||
curve=SECP256k1, name='example', force_detached=True)
|
||||
signed_copy = ClaimDict.deserialize(signed.serialized)
|
||||
fake_key = get_signer(SECP256k1).generate().private_key.to_pem()
|
||||
fake_cert = ClaimDict.generate_certificate(fake_key, curve=SECP256k1)
|
||||
self.assertRaises(ecdsa.keys.BadSignatureError, signed_copy.validate_signature,
|
||||
claim_address_2, fake_cert, 'example')
|
||||
|
||||
def test_fail_to_validate_ecdsa_sig_for_altered_claim(self):
|
||||
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
|
||||
altered = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_1, claim_id_1,
|
||||
curve=SECP256k1, name='example', force_detached=True)
|
||||
original_serialization = altered.serialized
|
||||
sd_hash = altered['stream']['source']['source']
|
||||
altered['stream']['source']['source'] = sd_hash[::-1]
|
||||
altered_serialization = altered.protobuf.SerializeToString()
|
||||
|
||||
# keep signature, but replace serialization with the altered claim (check signature.py for slice sizes)
|
||||
altered_copy = ClaimDict.deserialize(original_serialization[:85] + altered_serialization)
|
||||
self.assertRaises(ecdsa.keys.BadSignatureError, altered_copy.validate_signature,
|
||||
claim_address_1, cert, 'example')
|
||||
|
||||
|
||||
class TestMetadata(UnitTest):
|
||||
def test_fail_with_fake_sd_hash(self):
|
||||
claim = deepcopy(example_010)
|
||||
|
||||
@@ -12,6 +12,5 @@ setenv =
|
||||
TORBA_LEDGER=lbrynet.extras.wallet
|
||||
commands =
|
||||
orchstr8 download
|
||||
coverage run -p --source={envsitepackagesdir}/lbrynet -m unittest integration.wallet.test_transactions.BasicTransactionTest
|
||||
coverage run -p --source={envsitepackagesdir}/lbrynet -m twisted.trial --reactor=asyncio integration.cli
|
||||
coverage run -p --source={envsitepackagesdir}/lbrynet -m twisted.trial --reactor=asyncio integration.wallet.test_commands
|
||||
coverage run -p --source={envsitepackagesdir}/lbrynet -m unittest discover -vv integration.wallet
|
||||
coverage run -p --source={envsitepackagesdir}/lbrynet -m unittest discover -vv integration.cli
|
||||
Reference in new issue
Block a user