Compare commits

...
28 Commits
Author SHA1 Message Date
Victor Shyba 574c2ce794 adds integration test for release_time 2019-01-23 11:16:43 -03:00
Victor Shyba 7f5541eb5c adds release_time parameter to RPC publish 2019-01-23 11:16:43 -03:00
Victor Shyba 7b49241c38 adjust balance assertions for integration test using new signature model due new fees from new size 2019-01-23 11:16:43 -03:00
Victor Shyba da716b2e3d remove new signature model integration test as its the default now 2019-01-23 11:16:43 -03:00
Victor Shyba a144fae8b2 activate new signature model 2019-01-23 11:16:43 -03:00
Victor Shyba 0d90b82287 clear releaseTime when not set 2019-01-23 11:16:43 -03:00
Victor Shyba 567c23e10f regen for metadata releaseTime field 2019-01-23 11:16:43 -03:00
Victor Shyba 16970b7d37 regen proto from lbry/types, removes local .proto files 2019-01-23 11:16:43 -03:00
Lex Berezhny 8593cd362a check darwin before windows because darwin has win, face plam! 2019-01-22 18:04:17 -05:00
Lex Berezhny 2dbf96da81 blob downer test fixes 2019-01-22 18:04:17 -05:00
Lex Berezhny 83c7797187 pylint 2019-01-22 18:04:17 -05:00
Lex Berezhny 6606deafa5 fixed after testing with desktop app 2019-01-22 18:04:17 -05:00
Lex Berezhny 19b0d59159 config options api_host and api_port changed to just api 2019-01-22 18:04:17 -05:00
Lex Berezhny 9ccb3fa2a3 unit/test_cli tests 2019-01-22 18:04:17 -05:00
Lex Berezhny 48533df523 fix integration tests 2019-01-22 18:04:17 -05:00
Lex Berezhny 1ae1137005 fixing integration tests 2019-01-22 18:04:17 -05:00
Lex Berezhny d5f2832b2f pylint 2019-01-22 18:04:17 -05:00
Lex Berezhny d0230b4893 command line fixes 2019-01-22 18:04:17 -05:00
Lex Berezhny a6cd53b97c test_Daemon.py uses /tmp for data_dir 2019-01-22 18:04:17 -05:00
Lex Berezhny 7a28171a72 conf no longer global 2019-01-22 18:04:17 -05:00
Lex Berezhny 150bcb1116 new configuration manager, first pass 2019-01-22 18:04:17 -05:00
Jack Robison 6fe9c5bf00 Merge pull request #1794 from lbryio/async-exchange-rate-manager
refactor exchange rate manager to use asyncio
2019-01-22 12:05:07 -05:00
Jack Robison 3a86fc6680 refactor exchange rate manager to use asyncio 2019-01-22 11:34:49 -05:00
Jack Robison 2805cbb331 refactor loggly handler to use asyncio 2019-01-22 10:52:50 -05:00
Jack Robison 68d7a7014c Merge pull request #1792 from lbryio/delete-unused-scripts
delete unused scripts
2019-01-22 09:53:24 -05:00
Jack Robison 883fa267a5 delete unused scripts 2019-01-22 09:33:42 -05:00
Lex Berezhny 5f2c3ac965 dont run customlogger tests 2019-01-19 16:32:11 -05:00
Lex Berezhny bf8e32ebd2 remove custom logging 2019-01-19 16:30:19 -05:00
95 changed files with 1641 additions and 3715 deletions

No files matched your search

+2 -1
View File
@@ -24,8 +24,9 @@ jobs:
- pip install git+https://github.com/lbryio/torba.git#egg=torba
- pip install -e .[test]
script:
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.schema tests.unit.wallet tests.unit.test_customLogger tests.unit.components
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.core tests.unit.cryptstream tests.unit.database tests.unit.dht tests.unit.lbryfilemanager tests.unit.lbrynet_daemon tests.unit.schema tests.unit.wallet tests.unit.components tests.unit.test_conf
- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.test_cli
#- HOME=/tmp coverage run -p --source=lbrynet -m twisted.trial --reactor=asyncio tests.unit.analytics
after_success:
- coverage combine
- bash <(curl -s https://codecov.io/bash)
-6
View File
@@ -1,10 +1,4 @@
import logging
from lbrynet.custom_logger import install_logger
__name__ = "lbrynet"
__version__ = "0.30.5"
version = tuple(__version__.split('.'))
install_logger()
logging.getLogger(__name__).addHandler(logging.NullHandler())
+9 -10
View File
@@ -5,13 +5,12 @@ import logging
from binascii import hexlify, unhexlify
from twisted.internet import defer
from lbrynet import conf
from lbrynet.conf import Config
from lbrynet.extras.compat import f2d
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
from lbrynet.utils import short_hash
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileDownloader
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
from lbrynet.p2p.StreamDescriptor import save_sd_info
@@ -35,12 +34,12 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
STATUS_STOPPED = "stopped"
STATUS_FINISHED = "finished"
def __init__(self, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage, lbry_file_manager,
payment_rate_manager, wallet, download_directory, file_name, stream_name, sd_hash, key,
suggested_file_name, download_mirrors=None):
def __init__(self, conf: Config, rowid, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
lbry_file_manager, payment_rate_manager, wallet, download_directory, file_name, stream_name,
sd_hash, key, suggested_file_name, download_mirrors=None):
super().__init__(
stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
download_directory, key, stream_name, file_name
conf, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager,
wallet, download_directory, key, stream_name, file_name
)
self.sd_hash = sd_hash
self.rowid = rowid
@@ -56,9 +55,9 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
self.channel_name = None
self.metadata = None
self.mirror = None
if download_mirrors or conf.settings['download_mirrors']:
if download_mirrors or conf.download_mirrors:
self.mirror = HTTPBlobDownloader(
self.blob_manager, servers=download_mirrors or conf.settings['download_mirrors']
self.blob_manager, servers=download_mirrors or conf.download_mirrors
)
def set_claim_info(self, claim_info):
@@ -100,7 +99,7 @@ class ManagedEncryptedFileDownloader(EncryptedFileSaver):
if self.mirror:
self.mirror.stop()
# EncryptedFileSaver deletes metadata when it's stopped. We don't want that here.
yield EncryptedFileDownloader.stop(self, err=err)
yield super().stop(err)
if change_status is True:
status = yield self._save_status()
defer.returnValue(status)
+9 -5
View File
@@ -3,11 +3,12 @@ Keep track of which LBRY Files are downloading and store their LBRY File specifi
"""
import os
import logging
import random
from binascii import hexlify, unhexlify
from twisted.internet import defer, task, reactor
from twisted.python.failure import Failure
from lbrynet import conf
from lbrynet.conf import Config
from lbrynet.extras.compat import f2d
from lbrynet.extras.reflector.reupload import reflect_file
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
@@ -28,9 +29,11 @@ class EncryptedFileManager:
# when reflecting files, reflect up to this many files at a time
CONCURRENT_REFLECTS = 5
def __init__(self, peer_finder, rate_limiter, blob_manager, wallet, payment_rate_manager, storage, sd_identifier):
self.auto_re_reflect = conf.settings['reflect_uploads'] and conf.settings['auto_re_reflect_interval'] > 0
self.auto_re_reflect_interval = conf.settings['auto_re_reflect_interval']
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, wallet,
payment_rate_manager, storage, sd_identifier):
self.conf = conf
self.auto_re_reflect = conf.reflect_uploads and conf.auto_re_reflect_interval > 0
self.auto_re_reflect_interval = conf.auto_re_reflect_interval
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.blob_manager = blob_manager
@@ -78,6 +81,7 @@ class EncryptedFileManager:
def _get_lbry_file(self, rowid, stream_hash, payment_rate_manager, sd_hash, key,
stream_name, file_name, download_directory, suggested_file_name, download_mirrors=None):
return ManagedEncryptedFileDownloader(
self.conf,
rowid,
stream_hash,
self.peer_finder,
@@ -239,7 +243,7 @@ class EncryptedFileManager:
sd_hashes_to_reflect = yield f2d(self.storage.get_streams_to_re_reflect())
for lbry_file in self.lbry_files:
if lbry_file.sd_hash in sd_hashes_to_reflect:
ds.append(sem.run(reflect_file, lbry_file))
ds.append(sem.run(reflect_file, lbry_file, random.choice(self.conf.reflector_servers)))
yield defer.DeferredList(ds)
@defer.inlineCallbacks
+3 -2
View File
@@ -3,6 +3,7 @@ from binascii import unhexlify
from twisted.internet import defer
from twisted.python.failure import Failure
from lbrynet.conf import Config
from lbrynet.p2p.client.BlobRequester import BlobRequester
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
from lbrynet.p2p.client.DownloadManager import DownloadManager
@@ -37,7 +38,7 @@ class CryptStreamDownloader:
#implements(IStreamDownloader)
def __init__(self, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, payment_rate_manager, wallet,
key, stream_name):
"""Initialize a CryptStreamDownloader
@@ -55,7 +56,7 @@ class CryptStreamDownloader:
@return:
"""
self.conf = conf
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.blob_manager = blob_manager
+14 -15
View File
@@ -4,6 +4,7 @@ import traceback
from binascii import hexlify, unhexlify
from twisted.internet import defer, threads
from lbrynet.conf import Config
from lbrynet.extras.compat import f2d
from lbrynet.p2p.StreamDescriptor import save_sd_info
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
@@ -18,9 +19,9 @@ log = logging.getLogger(__name__)
class EncryptedFileDownloader(CryptStreamDownloader):
"""Classes which inherit from this class download LBRY files"""
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager,
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager,
storage, payment_rate_manager, wallet, key, stream_name, file_name):
super().__init__(peer_finder, rate_limiter, blob_manager,
super().__init__(conf, peer_finder, rate_limiter, blob_manager,
payment_rate_manager, wallet, key, stream_name)
self.stream_hash = stream_hash
self.storage = storage
@@ -37,7 +38,7 @@ class EncryptedFileDownloader(CryptStreamDownloader):
def stop(self, err=None):
self._close_output()
return CryptStreamDownloader.stop(self, err=err)
return super().stop(err=err)
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading,
@@ -97,7 +98,8 @@ class EncryptedFileDownloader(CryptStreamDownloader):
class EncryptedFileDownloaderFactory:
#implements(IStreamDownloaderFactory)
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet):
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet):
self.conf = conf
self.peer_finder = peer_finder
self.rate_limiter = rate_limiter
self.blob_manager = blob_manager
@@ -130,9 +132,9 @@ class EncryptedFileDownloaderFactory:
class EncryptedFileSaver(EncryptedFileDownloader):
def __init__(self, stream_hash, peer_finder, rate_limiter, blob_manager, storage, payment_rate_manager, wallet,
download_directory, key, stream_name, file_name):
super().__init__(stream_hash, peer_finder, rate_limiter,
def __init__(self, conf: Config, stream_hash, peer_finder, rate_limiter, blob_manager, storage,
payment_rate_manager, wallet, download_directory, key, stream_name, file_name):
super().__init__(conf, stream_hash, peer_finder, rate_limiter,
blob_manager, storage, payment_rate_manager,
wallet, key, stream_name, file_name)
self.download_directory = unhexlify(download_directory).decode()
@@ -142,10 +144,6 @@ class EncryptedFileSaver(EncryptedFileDownloader):
def __str__(self):
return str(self.file_written_to)
def stop(self, err=None):
d = EncryptedFileDownloader.stop(self, err=err)
return d
def _get_progress_manager(self, download_manager):
return FullStreamProgressManager(self._finished_downloading,
self.blob_manager,
@@ -182,8 +180,8 @@ class EncryptedFileSaver(EncryptedFileDownloader):
class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
def __init__(self, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
super().__init__(peer_finder, rate_limiter, blob_manager, storage, wallet)
def __init__(self, conf: Config, peer_finder, rate_limiter, blob_manager, storage, wallet, download_directory):
super().__init__(conf, peer_finder, rate_limiter, blob_manager, storage, wallet)
self.download_directory = hexlify(download_directory.encode())
def _make_downloader(self, stream_hash, payment_rate_manager, stream_info):
@@ -191,8 +189,9 @@ class EncryptedFileSaverFactory(EncryptedFileDownloaderFactory):
key = stream_info.raw_info['key']
suggested_file_name = stream_info.raw_info['suggested_file_name']
return EncryptedFileSaver(
stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage, payment_rate_manager,
self.wallet, self.download_directory, key=key, stream_name=stream_name, file_name=suggested_file_name
self.conf, stream_hash, self.peer_finder, self.rate_limiter, self.blob_manager, self.storage,
payment_rate_manager, self.wallet, self.download_directory, key=key, stream_name=stream_name,
file_name=suggested_file_name
)
@staticmethod
+455 -606
View File
File diff suppressed because it is too large. Load diff
-109
View File
@@ -1,109 +0,0 @@
import os
import sys
import inspect
import logging
TRACE = 5
####
# This code is copied from logging/__init__.py in the python source code
####
#
# _srcfile is used when walking the stack to check when we've got the first
# caller stack frame.
#
if hasattr(sys, 'frozen'): # support for py2exe
_srcfile = "logging{}__init__{}".format(os.sep, __file__[-4:])
elif __file__[-4:].lower() in ['.pyc', '.pyo']:
_srcfile = __file__[:-4] + '.py'
else:
_srcfile = __file__
_srcfile = os.path.normcase(_srcfile)
def findCaller(srcfile=None):
"""Returns the filename, line number and function name of the caller"""
srcfile = srcfile or _srcfile
f = inspect.currentframe()
# On some versions of IronPython, currentframe() returns None if
# IronPython isn't run with -X:Frames.
if f is not None:
f = f.f_back
rv = "(unknown file)", 0, "(unknown function)"
while hasattr(f, "f_code"):
co = f.f_code
filename = os.path.normcase(co.co_filename)
# ignore any function calls that are in this file
if filename == srcfile:
f = f.f_back
continue
rv = (filename, f.f_lineno, co.co_name)
break
return rv
###
class Logger(logging.Logger):
"""A logger that has an extra `fail` method useful for handling twisted failures."""
def fail(self, callback=None, *args, **kwargs):
"""Returns a function to log a failure from an errback.
The returned function appends the error message and extracts
the traceback from `err`.
Example usage:
d.addErrback(log.fail(), 'This is an error message')
Although odd, making the method call is necessary to extract
out useful filename and line number information; otherwise the
reported values are from inside twisted's deferred handling
code.
Args:
callback: callable to call after making the log. The first argument
will be the `err` from the deferred
args: extra arguments to pass into `callback`
Returns: a function that takes the following arguments:
err: twisted.python.failure.Failure
msg: the message to log, using normal logging string iterpolation.
msg_args: the values to substitute into `msg`
msg_kwargs: set `level` to change from the default ERROR severity. Other
keywoards are treated as normal log kwargs.
"""
fn, lno, func = findCaller()
def _fail(err, msg, *msg_args, **msg_kwargs):
level = msg_kwargs.pop('level', logging.ERROR)
msg += ": %s"
msg_args += (err.getErrorMessage(),)
exc_info = (err.type, err.value, err.getTracebackObject())
record = self.makeRecord(
self.name, level, fn, lno, msg, msg_args, exc_info, func, msg_kwargs)
self.handle(record)
if callback:
try:
return callback(err, *args, **kwargs)
except Exception:
# log.fail is almost always called within an
# errback. If callback fails and we didn't catch
# the exception we would need to attach a second
# errback to deal with that, which we will almost
# never do and then we end up with an unhandled
# error that will get swallowed by twisted
self.exception('Failed to run callback')
return _fail
def trace(self, msg, *args, **kwargs):
if self.isEnabledFor(TRACE):
self._log(TRACE, msg, args, **kwargs)
def install_logger():
current = logging.getLoggerClass()
if current is not Logger:
logging.setLoggerClass(Logger)
logging.addLevelName(TRACE, 'TRACE')
+128 -176
View File
@@ -1,132 +1,47 @@
import sys
import os
import json
import asyncio
import argparse
import typing
# Set SSL_CERT_FILE env variable for Twisted SSL verification on Windows
# This needs to happen before anything else
if 'win' in sys.platform:
import certifi
os.environ['SSL_CERT_FILE'] = certifi.where()
from twisted.internet import asyncioreactor
if 'twisted.internet.reactor' not in sys.modules:
asyncioreactor.install()
else:
from twisted.internet import reactor
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
# pyinstaller hooks install the default reactor before
# any of our code runs, see kivy for similar problem:
# https://github.com/kivy/kivy/issues/4182
del sys.modules['twisted.internet.reactor']
asyncioreactor.install()
from twisted.internet import reactor
import logging
from aiohttp.client_exceptions import ClientConnectorError
from requests.exceptions import ConnectionError
from docopt import docopt
from textwrap import dedent
from lbrynet import conf, log_support, __name__ as lbrynet_name
from lbrynet.utils import check_connection, json_dumps_pretty
import aiohttp
from lbrynet.extras.compat import force_asyncioreactor_install
force_asyncioreactor_install()
from lbrynet import log_support, __name__ as lbrynet_name, __version__ as lbrynet_version
from lbrynet.extras.daemon.loggly_handler import get_loggly_handler
from lbrynet.conf import Config, CLIConfig
from lbrynet.utils import check_connection
from lbrynet.extras.daemon.Daemon import Daemon
from lbrynet.extras.daemon.DaemonConsole import main as daemon_console, LBRYAPIClient
from lbrynet.extras.system_info import get_platform
log = logging.getLogger(lbrynet_name)
optional_path_getter_type = typing.Optional[typing.Callable[[], str]]
log.addHandler(logging.NullHandler())
def start_daemon(settings: typing.Optional[typing.Dict] = None,
console_output: typing.Optional[bool] = True, verbose: typing.Optional[typing.List[str]] = None,
data_dir: typing.Optional[str] = None, wallet_dir: typing.Optional[str] = None,
download_dir: typing.Optional[str] = None):
settings = settings or {}
conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir)
for k, v in settings.items():
conf.settings.update({k, v}, data_types=(conf.TYPE_CLI,))
log_support.configure_logging(conf.settings.get_log_filename(), console_output, verbose)
log_support.configure_loggly_handler()
log.debug('Final Settings: %s', conf.settings.get_current_settings_dict())
log.info("Starting lbrynet-daemon from command line")
if check_connection():
daemon = Daemon()
reactor._asyncioEventloop.create_task(daemon.start_listening())
reactor.run()
else:
log.info("Not connected to internet, unable to start")
def display(data):
print(json.dumps(data, indent=2))
def start_daemon_with_cli_args(argv=None, data_dir: typing.Optional[str] = None,
wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None):
parser = argparse.ArgumentParser()
parser.add_argument(
"--http-auth", dest="useauth", action="store_true", default=False
)
parser.add_argument(
'--quiet', dest='quiet', action="store_true",
help='Disable all console output.'
)
parser.add_argument(
'--verbose', nargs="*",
help=('Enable debug output. Optionally specify loggers for which debug output '
'should selectively be applied.')
)
parser.add_argument(
'--version', action="store_true",
help='Show daemon version and quit'
)
args = parser.parse_args(argv)
settings = {}
if args.useauth:
print('--http-auth is no longer supported; an alternative solution using IPC is forthcoming.')
return
verbose = None
if args.verbose:
verbose = args.verbose
console_output = not args.quiet
if args.version:
print(json_dumps_pretty(get_platform()))
return
return start_daemon(settings, console_output, verbose, data_dir, wallet_dir, download_dir)
async def execute_command(method, params, data_dir: typing.Optional[str] = None,
wallet_dir: typing.Optional[str] = None, download_dir: typing.Optional[str] = None):
# this check if the daemon is running or not
conf.initialize_settings(data_dir=data_dir, wallet_dir=wallet_dir, download_dir=download_dir)
api = None
try:
api = await LBRYAPIClient.get_client()
await api.status()
except (ClientConnectorError, ConnectionError):
if api:
await api.session.close()
print("Could not connect to daemon. Are you sure it's running?")
return 1
# this actually executes the method
resp = await api.call(method, params)
try:
await api.session.close()
print(json.dumps(resp["result"], indent=2))
except KeyError:
if resp["error"]["code"] == -32500:
print(json.dumps(resp["error"], indent=2))
else:
print(json.dumps(resp["error"]["message"], indent=2))
async def execute_command(conf, method, params):
async with aiohttp.ClientSession() as session:
try:
message = {'method': method, 'params': params}
async with session.get(conf.api_connection_url, json=message) as resp:
try:
data = await resp.json()
if 'result' in data:
display(data['result'])
elif 'error' in data:
if 'message' in data['error']:
display(data['error']['message'])
else:
display(data['error'])
except Exception as e:
log.exception('Could not process response from server:', exc_info=e)
except aiohttp.ClientConnectionError:
print("Could not connect to daemon. Are you sure it's running?")
def print_help():
@@ -200,82 +115,119 @@ def set_kwargs(parsed_args):
return kwargs
def get_argument_parser():
main = argparse.ArgumentParser('lbrynet', add_help=False)
main.add_argument(
'--version', dest='cli_version', action="store_true",
help='Show lbrynet CLI version and exit.'
)
main.add_argument(
'-h', '--help', dest='help', action="store_true",
help='Show this help message and exit'
)
CLIConfig.contribute_args(main)
sub = main.add_subparsers(dest='command')
help = sub.add_parser('help', help='Detailed help for remote commands.')
help.add_argument('help_command', nargs='*')
start = sub.add_parser('start', help='Start lbrynet server.')
start.add_argument(
'--quiet', dest='quiet', action="store_true",
help='Disable all console output.'
)
start.add_argument(
'--verbose', nargs="*",
help=('Enable debug output. Optionally specify loggers for which debug output '
'should selectively be applied.')
)
Config.contribute_args(start)
api = Daemon.get_api_definitions()
for group in sorted(api):
group_command = sub.add_parser(group, help=api[group]['doc'])
group_command.set_defaults(group_doc=group_command)
if group in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
continue
commands = group_command.add_subparsers(dest='subcommand')
for command in api[group]['commands']:
commands.add_parser(command['name'], help=command['doc'].strip().splitlines()[0])
for deprecated in Daemon.deprecated_methods:
group_command = sub.add_parser(deprecated)
group_command.add_subparsers(dest='subcommand')
return main
def main(argv=None):
argv = argv or sys.argv[1:]
if not argv:
print_help()
return 1
parser = get_argument_parser()
args, command_args = parser.parse_known_args(argv)
dir_args = {}
if len(argv) >= 2:
dir_arg_keys = [
'data_dir',
'wallet_dir',
'download_directory'
]
conf = Config.create_from_arguments(args)
for arg in argv:
found_dir_arg = False
for key in dir_arg_keys:
if arg.startswith(f'--{key}='):
if key in dir_args:
print(f"Multiple values provided for '{key}' argument")
print_help()
return 1
dir_args[key] = os.path.expanduser(os.path.expandvars(arg.lstrip(f'--{key}=')))
found_dir_arg = True
if not found_dir_arg:
break
argv = argv[len(dir_args):]
if args.cli_version:
print(f"{lbrynet_name} {lbrynet_version}")
return 0
data_dir = dir_args.get('data_dir')
wallet_dir = dir_args.get('wallet_dir')
download_dir = dir_args.get('download_directory')
elif args.command == 'start':
for k, v in dir_args.items():
if not os.path.isdir(v):
print(f"'{data_dir}' is not a directory, cannot use it for {k}")
return 1
log_support.configure_logging(conf.log_file_path, not args.quiet, args.verbose)
method, args = argv[0], argv[1:]
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
log.addHandler(loggly_handler)
if method in ['help', '--help', '-h']:
if len(args) == 1:
print_help_for_command(args[0])
log.debug('Final Settings: %s', conf.settings_dict)
log.info("Starting lbrynet-daemon from command line")
daemon = Daemon(conf)
if check_connection():
from twisted.internet import reactor
reactor._asyncioEventloop.create_task(daemon.start())
reactor.run()
else:
print_help()
return 0
log.info("Not connected to internet, unable to start")
elif method in ['version', '--version', '-v']:
print("{lbrynet_name} {lbrynet_version}".format(
lbrynet_name=lbrynet_name, **get_platform()
))
return 0
elif args.command == 'help':
elif method == 'start':
sys.exit(start_daemon_with_cli_args(args, data_dir, wallet_dir, download_dir))
elif method == 'console':
sys.exit(daemon_console())
elif method not in Daemon.callable_methods:
if method not in Daemon.deprecated_methods:
print(f'{method} is not a valid command.')
return 1
new_method = Daemon.deprecated_methods[method].new_command
if new_method is None:
print(f"{method} is permanently deprecated and does not have a replacement command.")
if args.help_command:
method = '_'.join(args.help_command)
else:
parser.print_help()
return 0
print(f"{method} is deprecated, using {new_method}.")
method = new_method
if method not in Daemon.callable_methods:
print('Invalid command name: {method}')
return 1
fn = Daemon.callable_methods[method]
parsed = docopt(fn.__doc__, args)
params = set_kwargs(parsed)
loop = asyncio.get_event_loop()
loop.run_until_complete(execute_command(method, params, data_dir, wallet_dir, download_dir))
fn = Daemon.callable_methods[method]
print(fn.__doc__)
elif args.command is not None:
if args.command in ('status', 'publish', 'version', 'help', 'wallet_balance', 'get'):
method = args.command
elif args.subcommand is not None:
method = f'{args.command}_{args.subcommand}'
else:
args.group_doc.print_help()
return 0
if method in Daemon.deprecated_methods:
new_method = Daemon.deprecated_methods[method].new_command
if new_method is None:
print(f"{method} is permanently deprecated and does not have a replacement command.")
return 0
print(f"{method} is deprecated, using {new_method}.")
method = new_method
fn = Daemon.callable_methods[method]
parsed = docopt(fn.__doc__, command_args)
params = set_kwargs(parsed)
loop = asyncio.get_event_loop()
loop.run_until_complete(execute_command(conf, method, params))
else:
parser.print_help()
return 0
+16 -1
View File
@@ -1,5 +1,19 @@
import asyncio
from twisted.internet import defer
def force_asyncioreactor_install():
import sys
from twisted.internet import asyncioreactor
if 'twisted.internet.reactor' not in sys.modules:
asyncioreactor.install()
else:
from twisted.internet import reactor
if not isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) and getattr(sys, 'frozen', False):
# pyinstaller hooks install the default reactor before
# any of our code runs, see kivy for similar problem:
# https://github.com/kivy/kivy/issues/4182
del sys.modules['twisted.internet.reactor']
asyncioreactor.install()
def d2f(deferred):
@@ -7,4 +21,5 @@ def d2f(deferred):
def f2d(future):
from twisted.internet import defer
return defer.Deferred.fromFuture(asyncio.ensure_future(future))
+3
View File
@@ -1,6 +1,8 @@
import logging
from twisted.internet import defer
from twisted._threads import AlreadyQuit
from lbrynet.conf import Config
from lbrynet.extras.daemon.ComponentManager import ComponentManager
log = logging.getLogger(__name__)
@@ -26,6 +28,7 @@ class Component(metaclass=ComponentType):
component_name = None
def __init__(self, component_manager):
self.conf: Config = component_manager.conf
self.component_manager = component_manager
self._running = False
+4 -1
View File
@@ -3,6 +3,7 @@ import logging
from lbrynet.p2p.Error import ComponentStartConditionNotMet
from lbrynet.extras.daemon.PeerManager import PeerManager
from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
from lbrynet.conf import Config
log = logging.getLogger(__name__)
@@ -34,8 +35,9 @@ class RequiredCondition(metaclass=RequiredConditionType):
class ComponentManager:
default_component_classes = {}
def __init__(self, reactor=None, analytics_manager=None, skip_components=None,
def __init__(self, conf: Config, reactor=None, analytics_manager=None, skip_components=None,
peer_manager=None, peer_finder=None, **override_components):
self.conf = conf
self.skip_components = skip_components or []
self.reactor = reactor
self.component_classes = {}
@@ -55,6 +57,7 @@ class ComponentManager:
for component_class in self.component_classes.values():
self.components.add(component_class(self))
self.daemon = None
def evaluate_condition(self, condition_name):
if condition_name not in RegisteredConditions.conditions:
+40 -40
View File
@@ -14,15 +14,15 @@ from aioupnp.upnp import UPnP
from aioupnp.fault import UPnPError
import lbrynet.schema
from lbrynet import conf
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
from lbrynet.extras.compat import d2f
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.Component import Component
from lbrynet.extras.daemon.ExchangeRateManager import ExchangeRateManager
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
@@ -90,25 +90,24 @@ class DatabaseComponent(Component):
def get_current_db_revision():
return 9
@staticmethod
def get_revision_filename():
return conf.settings.get_db_revision_filename()
@property
def revision_filename(self):
return self.component_manager.daemon.db_revision_file_path
@staticmethod
def _write_db_revision_file(version_num):
with open(conf.settings.get_db_revision_filename(), mode='w') as db_revision:
def _write_db_revision_file(self, version_num):
with open(self.revision_filename, mode='w') as db_revision:
db_revision.write(str(version_num))
async def start(self):
# check directories exist, create them if they don't
log.info("Loading databases")
if not os.path.exists(self.get_revision_filename()):
if not os.path.exists(self.revision_filename):
log.warning("db_revision file not found. Creating it")
self._write_db_revision_file(self.get_current_db_revision())
# check the db migration and run any needed migrations
with open(self.get_revision_filename(), "r") as revision_read_handle:
with open(self.revision_filename, "r") as revision_read_handle:
old_revision = int(revision_read_handle.read().strip())
if old_revision > self.get_current_db_revision():
@@ -119,13 +118,13 @@ class DatabaseComponent(Component):
from lbrynet.extras.daemon.migrator import dbmigrator
log.info("Upgrading your databases (revision %i to %i)", old_revision, self.get_current_db_revision())
await asyncio.get_event_loop().run_in_executor(
None, dbmigrator.migrate_db, conf.settings.data_dir, old_revision, self.get_current_db_revision()
None, dbmigrator.migrate_db, self.conf.data_dir, old_revision, self.get_current_db_revision()
)
self._write_db_revision_file(self.get_current_db_revision())
log.info("Finished upgrading the databases.")
self.storage = SQLiteStorage(
os.path.join(conf.settings.data_dir, "lbrynet.sqlite")
self.conf, os.path.join(self.conf.data_dir, "lbrynet.sqlite")
)
await self.storage.open()
@@ -143,9 +142,9 @@ class HeadersComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.headers_dir = os.path.join(conf.settings.wallet_dir, 'lbc_mainnet')
self.headers_dir = os.path.join(self.conf.wallet_dir, 'lbc_mainnet')
self.headers_file = os.path.join(self.headers_dir, 'headers')
self.old_file = os.path.join(conf.settings.wallet_dir, 'blockchain_headers')
self.old_file = os.path.join(self.conf.wallet_dir, 'blockchain_headers')
self._downloading_headers = None
self._headers_progress_percent = 0
@@ -204,8 +203,8 @@ class HeadersComponent(Component):
async def get_remote_height(self):
ledger = SimpleNamespace()
ledger.config = {
'default_servers': conf.settings['lbryum_servers'],
'data_path': conf.settings.wallet_dir
'default_servers': self.conf.lbryum_servers,
'data_path': self.conf.wallet_dir
}
net = Network(ledger)
first_connection = net.on_connected.first
@@ -216,10 +215,10 @@ class HeadersComponent(Component):
return remote_height
async def should_download_headers_from_s3(self):
if conf.settings['blockchain_name'] != "lbrycrd_main":
if self.conf.blockchain_name != "lbrycrd_main":
return False
self._check_header_file_integrity()
s3_headers_depth = conf.settings['s3_headers_depth']
s3_headers_depth = self.conf.s3_headers_depth
if not s3_headers_depth:
return False
local_height = self.local_header_file_height()
@@ -231,10 +230,10 @@ class HeadersComponent(Component):
def _check_header_file_integrity(self):
# TODO: temporary workaround for usability. move to txlbryum and check headers instead of file integrity
if conf.settings['blockchain_name'] != "lbrycrd_main":
if self.conf.blockchain_name != "lbrycrd_main":
return
hashsum = sha256()
checksum_height, checksum = conf.settings['HEADERS_FILE_SHA256_CHECKSUM']
checksum_height, checksum = HEADERS_FILE_SHA256_CHECKSUM
checksum_length_in_bytes = checksum_height * HEADER_SIZE
if self.local_header_file_size() < checksum_length_in_bytes:
return
@@ -252,7 +251,6 @@ class HeadersComponent(Component):
headers_file.truncate(checksum_length_in_bytes)
async def start(self):
conf.settings.ensure_wallet_dir()
if not os.path.exists(self.headers_dir):
os.mkdir(self.headers_dir)
if os.path.exists(self.old_file):
@@ -297,11 +295,10 @@ class WalletComponent(Component):
}
async def start(self):
conf.settings.ensure_wallet_dir()
log.info("Starting torba wallet")
storage = self.component_manager.get_component(DATABASE_COMPONENT)
lbrynet.schema.BLOCKCHAIN_NAME = conf.settings['blockchain_name']
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(conf.settings, storage)
lbrynet.schema.BLOCKCHAIN_NAME = self.conf.blockchain_name
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf, storage)
self.wallet_manager.old_db = storage
await self.wallet_manager.start()
@@ -329,7 +326,7 @@ class BlobComponent(Component):
dht_node = self.component_manager.get_component(DHT_COMPONENT)
if dht_node:
datastore = dht_node._dataStore
self.blob_manager = DiskBlobManager(os.path.join(conf.settings.data_dir, "blobfiles"), storage, datastore)
self.blob_manager = DiskBlobManager(os.path.join(self.conf.data_dir, "blobfiles"), storage, datastore)
return self.blob_manager.setup()
def stop(self):
@@ -359,15 +356,15 @@ class DHTComponent(Component):
async def get_status(self):
return {
'node_id': binascii.hexlify(conf.settings.get_node_id()),
'node_id': binascii.hexlify(self.component_manager.daemon.node_id),
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts)
}
async def start(self):
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", conf.settings["peer_port"])
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", conf.settings["dht_node_port"])
node_id = conf.settings.get_node_id()
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
node_id = self.component_manager.daemon.node_id
if node_id is None:
node_id = generate_id()
external_ip = self.upnp_component.external_ip
@@ -379,13 +376,13 @@ class DHTComponent(Component):
self.dht_node = Node(
node_id=node_id,
udpPort=conf.settings['dht_node_port'],
udpPort=self.conf.dht_node_port,
externalUDPPort=self.external_udp_port,
externalIP=external_ip,
peerPort=self.external_peer_port
)
await d2f(self.dht_node.start(conf.settings['known_dht_nodes'], block_on_join=False))
await d2f(self.dht_node.start(self.conf.known_dht_nodes, block_on_join=False))
log.info("Started the dht")
def stop(self):
@@ -407,7 +404,7 @@ class HashAnnouncerComponent(Component):
async def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.hash_announcer = DHTHashAnnouncer(dht_node, storage)
self.hash_announcer = DHTHashAnnouncer(self.conf, dht_node, storage)
self.hash_announcer.start()
def stop(self):
@@ -484,19 +481,22 @@ class FileManagerComponent(Component):
sd_identifier = StreamDescriptorIdentifier()
add_lbry_file_to_sd_identifier(sd_identifier)
file_saver_factory = EncryptedFileSaverFactory(
self.conf,
self.component_manager.peer_finder,
rate_limiter,
blob_manager,
storage,
wallet,
conf.settings.download_dir
self.conf.download_dir
)
sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
log.info('Starting the file manager')
self.file_manager = EncryptedFileManager(self.component_manager.peer_finder, rate_limiter, blob_manager, wallet,
payment_rate_manager, storage, sd_identifier)
self.file_manager = EncryptedFileManager(
self.conf, self.component_manager.peer_finder, rate_limiter,
blob_manager, wallet, payment_rate_manager, storage, sd_identifier
)
return self.file_manager.setup()
def stop(self):
@@ -519,7 +519,7 @@ class PeerProtocolServerComponent(Component):
async def start(self):
wallet = self.component_manager.get_component(WALLET_COMPONENT)
upnp = self.component_manager.get_component(UPNP_COMPONENT)
peer_port = conf.settings['peer_port']
peer_port = self.conf.peer_port
query_handlers = {
handler.get_primary_query_identifier(): handler for handler in [
BlobRequestHandlerFactory(
@@ -560,7 +560,7 @@ class ReflectorComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.reflector_server_port = conf.settings['reflector_port']
self.reflector_server_port = self.conf.reflector_port
self.reflector_server = None
@property
@@ -591,9 +591,9 @@ class UPnPComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self._int_peer_port = conf.settings['peer_port']
self._int_dht_node_port = conf.settings['dht_node_port']
self.use_upnp = conf.settings['use_upnp']
self._int_peer_port = self.conf.peer_port
self._int_dht_node_port = self.conf.dht_node_port
self.use_upnp = self.conf.use_upnp
self.upnp = None
self.upnp_redirects = {}
self.external_ip = None
+173 -109
View File
@@ -2,6 +2,7 @@ import os
import requests
import urllib
import textwrap
import random
from typing import Callable, Optional, List
from operator import itemgetter
@@ -45,6 +46,7 @@ from lbrynet.extras.daemon.ComponentManager import ComponentManager
from lbrynet.extras.looping_call_manager import LoopingCallManager
from lbrynet.p2p.Error import ComponentsNotStarted, ComponentStartConditionNotMet
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
import base58
import asyncio
import logging
@@ -56,7 +58,7 @@ from twisted.internet import defer
from lbrynet import utils
from lbrynet.extras.daemon.undecorated import undecorated
from lbrynet import conf
from lbrynet.conf import Config, Setting, SLACK_WEBHOOK
from aiohttp import web
@@ -211,16 +213,6 @@ def sort_claim_results(claims):
return claims
def is_first_run():
if os.path.isfile(conf.settings.get_db_revision_filename()):
return False
if os.path.isfile(os.path.join(conf.settings.data_dir, 'lbrynet.sqlite')):
return False
if os.path.isfile(os.path.join(conf.settings.wallet_dir, 'blockchain_headers')):
return False
return True
DHT_HAS_CONTACTS = "dht_has_contacts"
WALLET_IS_UNLOCKED = "wallet_is_unlocked"
@@ -354,26 +346,29 @@ class Daemon(metaclass=JSONRPCServerType):
allowed_during_startup = []
def __init__(self, analytics_manager=None, component_manager=None):
to_skip = conf.settings['components_to_skip']
if 'reflector' not in to_skip and not conf.settings['run_reflector_server']:
def __init__(self, conf: Config, component_manager: ComponentManager = None):
self.conf = conf
to_skip = conf.components_to_skip
if 'reflector' not in to_skip and not conf.run_reflector_server:
to_skip.append('reflector')
looping_calls = {
Checker.INTERNET_CONNECTION[0]: (LoopingCall(CheckInternetConnection(self)),
Checker.INTERNET_CONNECTION[1])
}
self.analytics_manager = analytics_manager or analytics.Manager.new_instance()
self._node_id = None
self._installation_id = None
self.session_id = base58.b58encode(utils.generate_id()).decode()
self.analytics_manager = analytics.Manager(conf, self.installation_id, self.session_id)
self.component_manager = component_manager or ComponentManager(
analytics_manager=self.analytics_manager,
skip_components=to_skip or [],
conf, analytics_manager=self.analytics_manager, skip_components=to_skip or []
)
self.component_manager.daemon = self
self.looping_call_manager = LoopingCallManager({n: lc for n, (lc, t) in (looping_calls or {}).items()})
self._looping_call_times = {n: t for n, (lc, t) in (looping_calls or {}).items()}
self.listening_port = None
self._component_setup_task = None
self.announced_startup = False
self.sessions = {}
self.is_first_run = is_first_run()
# TODO: move this to a component
self.connected_to_internet = True
@@ -402,17 +397,83 @@ class Daemon(metaclass=JSONRPCServerType):
self.handler = self.app.make_handler()
self.server = None
async def start_listening(self):
@classmethod
def get_api_definitions(cls):
groups = {}
for method in dir(cls):
if method.startswith('jsonrpc_'):
parts = method.split('_', 2)
group = command = parts[1]
if len(parts) == 3:
command = parts[2]
group_dict = {'doc': getattr(cls, f'{group.upper()}_DOC', ''), 'commands': []}
groups.setdefault(group, group_dict)['commands'].append({
'name': command,
'doc': getattr(cls, method).__doc__
})
del groups['commands']
del groups['help']
return groups
@property
def db_revision_file_path(self):
return os.path.join(self.conf.data_dir, 'db_revision')
@property
def installation_id(self):
install_id_filename = os.path.join(self.conf.data_dir, "install_id")
if not self._installation_id:
if os.path.isfile(install_id_filename):
with open(install_id_filename, "r") as install_id_file:
self._installation_id = str(install_id_file.read()).strip()
if not self._installation_id:
self._installation_id = base58.b58encode(utils.generate_id()).decode()
with open(install_id_filename, "w") as install_id_file:
install_id_file.write(self._installation_id)
return self._installation_id
@property
def node_id(self):
node_id_filename = os.path.join(self.conf.data_dir, "node_id")
if not self._node_id:
if os.path.isfile(node_id_filename):
with open(node_id_filename, "r") as node_id_file:
self._node_id = base58.b58decode(str(node_id_file.read()).strip())
if not self._node_id:
self._node_id = utils.generate_id()
with open(node_id_filename, "w") as node_id_file:
node_id_file.write(base58.b58encode(self._node_id).decode())
return self._node_id
def ensure_data_dir(self):
# although there is a risk of a race condition here we don't
# expect there to be multiple processes accessing this
# directory so the risk can be ignored
if not os.path.isdir(self.conf.data_dir):
os.makedirs(self.conf.data_dir)
if not os.path.isdir(os.path.join(self.conf.data_dir, "blobfiles")):
os.makedirs(os.path.join(self.conf.data_dir, "blobfiles"))
return self.conf.data_dir
def ensure_wallet_dir(self):
if not os.path.isdir(self.conf.wallet_dir):
os.makedirs(self.conf.wallet_dir)
def ensure_download_dir(self):
if not os.path.isdir(self.conf.download_dir):
os.makedirs(self.conf.download_dir)
async def start(self):
try:
self.server = await asyncio.get_event_loop().create_server(
self.handler, conf.settings['api_host'], conf.settings['api_port']
self.handler, self.conf.api_host, self.conf.api_port
)
log.info('lbrynet API listening on TCP %s:%i', *self.server.sockets[0].getsockname()[:2])
await self.setup()
await self.analytics_manager.send_server_startup_success()
except OSError:
log.error('lbrynet API failed to bind TCP %s:%i for listening. Daemon is already running or this port is '
'already in use by another application.', conf.settings['api_host'], conf.settings['api_port'])
log.error('lbrynet API failed to bind TCP %s for listening. Daemon is already running or this port is '
'already in use by another application.', self.conf.api)
except defer.CancelledError:
log.info("shutting down before finished starting")
except Exception as err:
@@ -423,6 +484,10 @@ class Daemon(metaclass=JSONRPCServerType):
log.info("Starting lbrynet-daemon")
log.info("Platform: %s", json.dumps(system_info.get_platform()))
self.ensure_data_dir()
self.ensure_wallet_dir()
self.ensure_download_dir()
if not self.analytics_manager.is_started:
self.analytics_manager.start()
await self.analytics_manager.send_server_startup()
@@ -456,7 +521,7 @@ class Daemon(metaclass=JSONRPCServerType):
await self.handler.shutdown(60.0)
await self.app.cleanup()
if self.analytics_manager:
self.analytics_manager.shutdown()
self.analytics_manager.stop()
try:
self._component_setup_task.cancel()
except (AttributeError, asyncio.CancelledError):
@@ -673,10 +738,9 @@ class Daemon(metaclass=JSONRPCServerType):
await self.analytics_manager.send_download_started(download_id, name, claim_dict)
await self.analytics_manager.send_new_download_start(download_id, name, claim_dict)
self.streams[sd_hash] = GetStream(
self.file_manager.sd_identifier, self.wallet_manager, self.exchange_rate_manager, self.blob_manager,
self.component_manager.peer_finder, self.rate_limiter, self.payment_rate_manager, self.storage,
conf.settings['max_key_fee'], conf.settings['disable_max_key_fee'], conf.settings['data_rate'],
timeout
self.conf, self.file_manager.sd_identifier, self.wallet_manager, self.exchange_rate_manager,
self.blob_manager, self.component_manager.peer_finder, self.rate_limiter, self.payment_rate_manager,
self.storage, self.conf.max_key_fee, self.conf.disable_max_key_fee, self.conf.data_rate, timeout
)
try:
lbry_file, finished_deferred = await d2f(self.streams[sd_hash].start(
@@ -713,8 +777,8 @@ class Daemon(metaclass=JSONRPCServerType):
tx = await publisher.publish_stream(name, bid, claim_dict, stream_hash, claim_address)
else:
tx = await publisher.create_and_publish_stream(name, bid, claim_dict, file_path, claim_address)
if conf.settings['reflect_uploads']:
d = reupload.reflect_file(publisher.lbry_file)
if self.conf.reflect_uploads:
d = reupload.reflect_file(publisher.lbry_file, random.choice(self.conf.reflector_servers))
d.addCallbacks(lambda _: log.info("Reflected new publication to lbry://%s", name),
log.exception)
await self.analytics_manager.send_claim_action('publish')
@@ -733,9 +797,9 @@ class Daemon(metaclass=JSONRPCServerType):
if blob:
return self.blob_manager.get_blob(blob[0])
return await d2f(download_sd_blob(
sd_hash.decode(), self.blob_manager, self.component_manager.peer_finder, self.rate_limiter,
self.payment_rate_manager, self.wallet_manager, timeout=conf.settings['peer_search_timeout'],
download_mirrors=conf.settings['download_mirrors']
self.conf, sd_hash.decode(), self.blob_manager, self.component_manager.peer_finder, self.rate_limiter,
self.payment_rate_manager, self.wallet_manager, timeout=self.conf.peer_search_timeout,
download_mirrors=self.conf.download_mirrors
))
def get_or_download_sd_blob(self, sd_hash):
@@ -763,7 +827,7 @@ class Daemon(metaclass=JSONRPCServerType):
if self.payment_rate_manager.generous:
return 0.0
return size / (10 ** 6) * conf.settings['data_rate']
return size / (10 ** 6) * self.conf.data_rate
async def get_est_cost_using_known_size(self, uri, size):
"""
@@ -832,7 +896,7 @@ class Daemon(metaclass=JSONRPCServerType):
key = hexlify(lbry_file.key) if lbry_file.key else None
download_directory = lbry_file.download_directory
if not os.path.exists(download_directory):
download_directory = conf.settings.download_dir
download_directory = self.conf.download_dir
full_path = os.path.join(download_directory, lbry_file.file_name)
mime_type = guess_media_type(lbry_file.file_name)
if os.path.isfile(full_path):
@@ -925,15 +989,15 @@ class Daemon(metaclass=JSONRPCServerType):
return field, direction
def _get_single_peer_downloader(self):
downloader = SinglePeerDownloader()
downloader = SinglePeerDownloader(self.conf)
downloader.setup(self.wallet_manager)
return downloader
async def _blob_availability(self, blob_hash, search_timeout, blob_timeout, downloader=None):
if not downloader:
downloader = self._get_single_peer_downloader()
search_timeout = search_timeout or conf.settings['peer_search_timeout']
blob_timeout = blob_timeout or conf.settings['sd_download_timeout']
search_timeout = search_timeout or self.conf.peer_search_timeout
blob_timeout = blob_timeout or self.conf.sd_download_timeout
reachable_peers = []
unreachable_peers = []
try:
@@ -1000,7 +1064,6 @@ class Daemon(metaclass=JSONRPCServerType):
{
'installation_id': (str) installation id - base58,
'is_running': (bool),
'is_first_run': bool,
'skipped_components': (list) [names of skipped components (str)],
'startup_status': { Does not include components which have been skipped
'database': (bool),
@@ -1060,9 +1123,8 @@ class Daemon(metaclass=JSONRPCServerType):
connection_code = CONNECTION_STATUS_CONNECTED if self.connected_to_internet else CONNECTION_STATUS_NETWORK
response = {
'installation_id': conf.settings.installation_id,
'installation_id': self.installation_id,
'is_running': all(self.component_manager.get_components_status().values()),
'is_first_run': self.is_first_run,
'skipped_components': self.component_manager.skip_components,
'startup_status': self.component_manager.get_components_status(),
'connection_status': {
@@ -1118,16 +1180,25 @@ class Daemon(metaclass=JSONRPCServerType):
Returns:
(bool) true if successful
"""
platform_name = system_info.get_platform()['platform']
report_bug_to_slack(
message,
conf.settings.installation_id,
platform_name,
__version__
query = get_loggly_query_string(self.installation_id)
requests.post(
utils.deobfuscate(SLACK_WEBHOOK),
json.dumps({
"text": (
f"os: {platform_name}\n "
f"version: {__version__}\n"
f"<{query}|loggly>\n"
f"{message}"
)
})
)
return True
SETTINGS_DOC = """
Settings management.
"""
def jsonrpc_settings_get(self):
"""
Get daemon settings
@@ -1142,7 +1213,7 @@ class Daemon(metaclass=JSONRPCServerType):
(dict) Dictionary of daemon settings
See ADJUSTABLE_SETTINGS in lbrynet/conf.py for full list of settings
"""
return conf.settings.get_adjustable_settings_dict()
return self.conf.settings_dict
def jsonrpc_settings_set(self, **kwargs):
"""
@@ -1194,42 +1265,11 @@ class Daemon(metaclass=JSONRPCServerType):
Returns:
(dict) Updated dictionary of daemon settings
"""
# TODO: improve upon the current logic, it could be made better
new_settings = kwargs
setting_types = {
'download_directory': str,
'data_rate': float,
'download_timeout': int,
'peer_port': int,
'max_key_fee': dict,
'use_upnp': bool,
'run_reflector_server': bool,
'cache_time': int,
'reflect_uploads': bool,
'share_usage_data': bool,
'disable_max_key_fee': bool,
'peer_search_timeout': int,
'sd_download_timeout': int,
'auto_renew_claim_height_delta': int
}
for key, setting_type in setting_types.items():
if key in new_settings:
if isinstance(new_settings[key], setting_type):
conf.settings.update({key: new_settings[key]},
data_types=(conf.TYPE_RUNTIME, conf.TYPE_PERSISTED))
elif setting_type is dict and isinstance(new_settings[key], str):
decoded = json.loads(str(new_settings[key]))
conf.settings.update({key: decoded},
data_types=(conf.TYPE_RUNTIME, conf.TYPE_PERSISTED))
else:
converted = setting_type(new_settings[key])
conf.settings.update({key: converted},
data_types=(conf.TYPE_RUNTIME, conf.TYPE_PERSISTED))
conf.settings.save_conf_file_settings()
return conf.settings.get_adjustable_settings_dict()
with self.conf.update_config() as c:
for key, value in kwargs:
attr: Setting = getattr(type(c), key)
setattr(c, key, attr.deserialize(value))
return self.jsonrpc_settings_get()
def jsonrpc_help(self, command=None):
"""
@@ -1281,7 +1321,7 @@ class Daemon(metaclass=JSONRPCServerType):
@deprecated("account_balance")
def jsonrpc_wallet_balance(self, address=None):
pass
""" deprecated """
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
async def jsonrpc_wallet_send(self, amount, address=None, claim_id=None, account_id=None):
@@ -1349,6 +1389,10 @@ class Daemon(metaclass=JSONRPCServerType):
result = await self.jsonrpc_claim_tip(claim_id=claim_id, amount=amount, account_id=account_id)
return result
ACCOUNT_DOC = """
Account management.
"""
@requires("wallet")
def jsonrpc_account_list(self, account_id=None, confirmations=6,
include_claims=False, show_seed=False):
@@ -1728,6 +1772,10 @@ class Daemon(metaclass=JSONRPCServerType):
await self.analytics_manager.send_credits_sent()
return result
ADDRESS_DOC = """
Address management.
"""
@requires(WALLET_COMPONENT)
def jsonrpc_address_is_mine(self, address, account_id=None):
"""
@@ -1789,6 +1837,10 @@ class Daemon(metaclass=JSONRPCServerType):
"""
return self.get_account_or_default(account_id).receiving.get_or_create_usable_address()
FILE_DOC = """
File management.
"""
@requires(FILE_MANAGER_COMPONENT)
async def jsonrpc_file_list(self, sort=None, **kwargs):
"""
@@ -1882,6 +1934,10 @@ class Daemon(metaclass=JSONRPCServerType):
except UnknownNameError:
log.info('Name %s is not known', name)
CLAIM_DOC = """
Claim management.
"""
@requires(WALLET_COMPONENT)
async def jsonrpc_claim_show(self, txid=None, nout=None, claim_id=None):
"""
@@ -2061,7 +2117,7 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
timeout = timeout if timeout is not None else conf.settings['download_timeout']
timeout = timeout if timeout is not None else self.conf.download_timeout
parsed_uri = parse_lbry_uri(uri)
if parsed_uri.is_channel:
@@ -2214,6 +2270,10 @@ class Daemon(metaclass=JSONRPCServerType):
"""
return self.get_est_cost(uri, size)
CHANNEL_DOC = """
Channel management.
"""
@requires(WALLET_COMPONENT, conditions=[WALLET_IS_UNLOCKED])
async def jsonrpc_channel_new(self, channel_name, amount, account_id=None):
"""
@@ -2334,7 +2394,7 @@ class Daemon(metaclass=JSONRPCServerType):
description=None, author=None, language=None, license=None,
license_url=None, thumbnail=None, preview=None, nsfw=None, sources=None,
channel_name=None, channel_id=None, channel_account_id=None, account_id=None,
claim_address=None, change_address=None):
claim_address=None, change_address=None, release_time=None):
"""
Make a new name claim and publish associated data to lbrynet,
update over existing claim if user already has a claim for name.
@@ -2360,6 +2420,7 @@ class Daemon(metaclass=JSONRPCServerType):
[--channel_name=<channel_name>] [--channel_id=<channel_id>]
[--channel_account_id=<channel_account_id>...] [--account_id=<account_id>]
[--claim_address=<claim_address>] [--change_address=<change_address>]
[--release_time=<release_time>]
Options:
--name=<name> : (str) name of the content (can only consist of a-z A-Z 0-9 and -(dash))
@@ -2392,6 +2453,7 @@ class Daemon(metaclass=JSONRPCServerType):
--thumbnail=<thumbnail> : (str) thumbnail url
--preview=<preview> : (str) preview url
--nsfw=<nsfw> : (bool) whether the content is nsfw
--release_time=<release_time> : (int) seconds since Unix Epoch in GMT time. Can be negative.
--sources=<sources> : (str) {'lbry_sd_hash': sd_hash} specifies sd hash of file
--channel_name=<channel_name> : (str) name of the publisher channel name in the wallet
--channel_id=<channel_id> : (str) claim id of the publisher channel, does not check
@@ -2463,6 +2525,8 @@ class Daemon(metaclass=JSONRPCServerType):
metadata['preview'] = preview
if nsfw is not None:
metadata['nsfw'] = bool(nsfw)
if release_time is not None:
metadata['releaseTime'] = int(release_time)
metadata['version'] = '_0_1_0'
@@ -2849,6 +2913,10 @@ class Daemon(metaclass=JSONRPCServerType):
results[u]['claims_in_channel'] = resolved[u].get('claims_in_channel', [])
return results
CHANNEL_DOC = """
Transaction management.
"""
@requires(WALLET_COMPONENT)
def jsonrpc_transaction_list(self, account_id=None, page=None, page_size=None):
"""
@@ -2932,6 +3000,10 @@ class Daemon(metaclass=JSONRPCServerType):
"""
return self.wallet_manager.get_transaction(txid)
UTXO_DOC = """
Unspent transaction management.
"""
@requires(WALLET_COMPONENT)
def jsonrpc_utxo_list(self, account_id=None, page=None, page_size=None):
"""
@@ -3007,6 +3079,10 @@ class Daemon(metaclass=JSONRPCServerType):
"""
return self.wallet_manager.get_block(blockhash, height)
BLOB_DOC = """
Blob management.
"""
@requires(WALLET_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT, RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT,
conditions=[WALLET_IS_UNLOCKED])
async def jsonrpc_blob_get(self, blob_hash, timeout=None, encoding=None, payment_rate_manager=None):
@@ -3100,7 +3176,7 @@ class Daemon(metaclass=JSONRPCServerType):
err.trap(defer.TimeoutError)
return []
finished_deferred.addTimeout(timeout or conf.settings['peer_search_timeout'], self.dht_node.clock)
finished_deferred.addTimeout(timeout or self.conf.peer_search_timeout, self.dht_node.clock)
finished_deferred.addErrback(trap_timeout)
peers = await d2f(finished_deferred)
results = [
@@ -3175,7 +3251,7 @@ class Daemon(metaclass=JSONRPCServerType):
elif not lbry_files:
raise Exception('No file found')
return await d2f(reupload.reflect_file(
lbry_files[0], reflector_server=kwargs.get('reflector', None)
lbry_files[0], reflector_server=kwargs.get('reflector', random.choice(self.conf.reflector_servers))
))
@requires(BLOB_COMPONENT, WALLET_COMPONENT)
@@ -3253,8 +3329,9 @@ class Daemon(metaclass=JSONRPCServerType):
Returns:
(list) reflected blob hashes
"""
result = await d2f(reupload.reflect_blob_hashes(blob_hashes, self.blob_manager, reflector_server))
return result
return await d2f(reupload.reflect_blob_hashes(
blob_hashes, self.blob_manager, reflector_server or random.choice(self.conf.reflector_servers)
))
@requires(BLOB_COMPONENT)
async def jsonrpc_blob_reflect_all(self):
@@ -3271,7 +3348,9 @@ class Daemon(metaclass=JSONRPCServerType):
(bool) true if successful
"""
blob_hashes = await d2f(self.blob_manager.get_all_verified_blobs())
return await d2f(reupload.reflect_blob_hashes(blob_hashes, self.blob_manager))
return await d2f(reupload.reflect_blob_hashes(
blob_hashes, self.blob_manager, random.choice(self.conf.reflector_servers)
))
@requires(DHT_COMPONENT)
async def jsonrpc_peer_ping(self, node_id, address=None, port=None):
@@ -3427,8 +3506,8 @@ class Daemon(metaclass=JSONRPCServerType):
}
"""
search_timeout = search_timeout or conf.settings['peer_search_timeout']
blob_timeout = blob_timeout or conf.settings['sd_download_timeout']
search_timeout = search_timeout or self.conf.peer_search_timeout
blob_timeout = blob_timeout or self.conf.sd_download_timeout
response = {
'is_available': False,
@@ -3440,7 +3519,7 @@ class Daemon(metaclass=JSONRPCServerType):
'sd_blob_availability': {},
'head_blob_hash': None,
'head_blob_availability': {},
'use_upnp': conf.settings['use_upnp'],
'use_upnp': self.conf.use_upnp,
'upnp_redirect_is_set': len(self.upnp.upnp_redirects),
'error': None
}
@@ -3557,21 +3636,6 @@ def get_loggly_query_string(installation_id):
return base_loggly_search_url + data
def report_bug_to_slack(message, installation_id, platform_name, app_version):
webhook = utils.deobfuscate(conf.settings['SLACK_WEBHOOK'])
payload_template = "os: %s\n version: %s\n<%s|loggly>\n%s"
payload_params = (
platform_name,
app_version,
get_loggly_query_string(installation_id),
message
)
payload = {
"text": payload_template % payload_params
}
requests.post(webhook, json.dumps(payload))
def get_lbry_file_search_value(search_fields):
for searchtype in FileID:
value = search_fields.get(searchtype, None)
-284
View File
@@ -1,284 +0,0 @@
import sys
import code
import argparse
import asyncio
import logging.handlers
from twisted.internet import defer, reactor, threads
from aiohttp import client_exceptions
from lbrynet import utils, conf, log_support
from lbrynet.extras.daemon import analytics
from lbrynet.extras.daemon.Daemon import Daemon
import aiohttp
import logging
from urllib.parse import urlparse
log = logging.getLogger(__name__)
USER_AGENT = "AuthServiceProxy/0.1"
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
TWISTED_SESSION = "TWISTED_SESSION"
LBRY_SECRET = "LBRY_SECRET"
HTTP_TIMEOUT = 30
class JSONRPCException(Exception):
def __init__(self, rpc_error):
super().__init__()
self.error = rpc_error
class UnAuthAPIClient:
def __init__(self, host, port, session):
self.host = host
self.port = port
self.session = session
def __getattr__(self, method):
async def f(*args, **kwargs):
return await self.call(method, [args, kwargs])
return f
@classmethod
async def from_url(cls, url):
url_fragment = urlparse(url)
host = url_fragment.hostname
port = url_fragment.port
connector = aiohttp.TCPConnector()
session = aiohttp.ClientSession(connector=connector)
return cls(host, port, session)
async def call(self, method, params=None):
message = {'method': method, 'params': params}
async with self.session.get(conf.settings.get_api_connection_string(), json=message) as resp:
return await resp.json()
class LBRYAPIClient:
@staticmethod
def get_client(conf_path=None):
conf.conf_file = conf_path
if not conf.settings:
conf.initialize_settings()
return UnAuthAPIClient.from_url(conf.settings.get_api_connection_string())
if sys.platform.startswith('darwin') or sys.platform.startswith('linux'):
def color(msg, c="white"):
_colors = {
"normal": (0, 37),
"underlined": (2, 37),
"red": (1, 31),
"green": (1, 32),
"yellow": (1, 33),
"blue": (1, 33),
"magenta": (1, 34),
"cyan": (1, 35),
"white": (1, 36),
"grey": (1, 37)
}
i, j = _colors[c]
return "\033[%i;%i;40m%s\033[0m" % (i, j, msg)
logo = """\
╓▄█▄ç
,▄█▓▓▀▀▀▓▓▓▌▄,
▄▄▓▓▓▀¬ ╙▀█▓▓▓▄▄
,▄█▓▓▀▀ ^▀▀▓▓▓▌▄,
▄█▓▓█▀` ╙▀█▓▓▓▄▄
╓▄▓▓▓▀╙ ▀▀▓▓▓▌▄,
▄█▓▓█▀ ╙▀▓▓
╓▄▓▓▓▀` ▄█▓▓▓▀
▓▓█▀ ,▄▓▓▓▀╙
▓▓m ╟▌▄, ▄█▓▓█▀ ,,╓µ
▓▓m ^▀█▓▓▓▄▄ ╓▄▓▓▓▀╙ █▓▓▓▓▓▀
▓▓Q '▀▀▓▓▓▌▄, ,▄█▓▓█▀ ▄█▓▓▓▓▓▀
▀▓▓▓▌▄, ╙▀█▓▓█▄╗ ╓▄▓▓▓▀ ╓▄▓▓▓▀▀ ▀▀
╙▀█▓▓█▄╗ ^▀▀▓▓▓▌▄▄█▓▓▀▀ ▄█▓▓█▀`
'▀▀▓▓▓▌▄, ╙▀██▀` ╓▄▓▓▓▀╙
╙▀█▓▓█▄╥ ,▄█▓▓▀▀
└▀▀▓▓▓▌▄ ▄▒▓▓▓▀╙
╙▀█▓▓█▓▓▓▀▀
╙▀▀`
"""
else:
def color(msg, c=None):
return msg
logo = """\
'.
++++.
+++,;+++,
:+++ :+++:
+++ ,+++;
'++; .+++'
`+++ `++++
+++. `++++
;+++ ++++
+++ +++
+++: '+
,+++ +++
+++` +++:
`+' ,+++
`+ + +++
`+ +++ '++' :'+++:
`+ ++++ `+++ ++++
`+ ++++ +++. :+++'
`+, ++++ ;+++ +++++
`+++, ++++ +++ +++; +
,+++, ++++ +++: .+++
,+++: '++++++ +++`
,+++: '++ '++'
,+++: `+++
.+++; +++,
.+++; ;+++
.+++; +++
`+++++:
`++
"""
welcometext = """\
For a list of available commands:
>>>help()
To see the documentation for a given command:
>>>help("resolve")
To exit:
>>>exit()
"""
welcome = "{:*^60}\n".format(" Welcome to the lbrynet interactive console! ")
welcome += "\n".join([f"{w:<60}" for w in welcometext.splitlines()])
welcome += "\n%s" % ("*" * 60)
welcome = color(welcome, "grey")
banner = color(logo, "green") + color(welcome, "grey")
def get_methods(daemon):
locs = {}
def wrapped(name, fn):
client = LBRYAPIClient.get_client()
_fn = getattr(client, name)
_fn.__doc__ = fn.__doc__
return {name: _fn}
for method_name, method in daemon.callable_methods.items():
locs.update(wrapped(method_name, method))
return locs
def run_terminal(callable_methods, started_daemon, quiet=False):
locs = {}
locs.update(callable_methods)
def help(method_name=None):
if not method_name:
print("Available api functions: ")
for name in callable_methods:
print("\t%s" % name)
return
if method_name not in callable_methods:
print("\"%s\" is not a recognized api function")
return
print(callable_methods[method_name].__doc__)
return
locs.update({'help': help})
if started_daemon:
def exit(status=None):
if not quiet:
print("Stopping lbrynet-daemon...")
callable_methods['daemon_stop']()
return sys.exit(status)
locs.update({'exit': exit})
else:
def exit(status=None):
try:
reactor.callLater(0, reactor.stop)
except Exception as err:
print(f"error stopping reactor: {err}")
return sys.exit(status)
locs.update({'exit': exit})
code.interact(banner if not quiet else "", local=locs)
@defer.inlineCallbacks
def start_server_and_listen(use_auth, analytics_manager, quiet):
log_support.configure_console()
logging.getLogger("lbrynet").setLevel(logging.CRITICAL)
logging.getLogger("lbryum").setLevel(logging.CRITICAL)
logging.getLogger("requests").setLevel(logging.CRITICAL)
# TODO: turn this all into async. Until then this routine can't be called
# analytics_manager.send_server_startup()
yield Daemon().start_listening()
def threaded_terminal(started_daemon, quiet):
callable_methods = get_methods(Daemon)
d = threads.deferToThread(run_terminal, callable_methods, started_daemon, quiet)
d.addErrback(lambda err: err.trap(SystemExit))
d.addErrback(log.exception)
async def start_lbrynet_console(quiet, use_existing_daemon, useauth):
if not utils.check_connection():
print("Not connected to internet, unable to start")
raise Exception("Not connected to internet, unable to start")
if not quiet:
print("Starting lbrynet-console...")
try:
await LBRYAPIClient.get_client().status()
d = defer.succeed(False)
if not quiet:
print("lbrynet-daemon is already running, connecting to it...")
except client_exceptions.ClientConnectorError:
if not use_existing_daemon:
if not quiet:
print("Starting lbrynet-daemon...")
analytics_manager = analytics.Manager.new_instance()
d = start_server_and_listen(useauth, analytics_manager, quiet)
else:
raise Exception("cannot connect to an existing daemon instance, "
"and set to not start a new one")
d.addCallback(threaded_terminal, quiet)
d.addErrback(log.exception)
def main():
conf.initialize_settings()
parser = argparse.ArgumentParser(description="Launch lbrynet-daemon")
parser.add_argument(
"--use_existing_daemon",
help="Start lbrynet-daemon if it isn't already running",
action="store_true",
default=False,
dest="use_existing_daemon"
)
parser.add_argument(
"--quiet", dest="quiet", action="store_true", default=False
)
parser.add_argument(
"--http-auth", dest="useauth", action="store_true", default=conf.settings['use_auth_http']
)
args = parser.parse_args()
if args.useauth:
print('--http-auth is no longer supported; an alternative solution using IPC is forthcoming.')
return
loop = asyncio.get_event_loop()
loop.run_until_complete(start_lbrynet_console(args.quiet, args.use_existing_daemon, args.useauth))
reactor.run()
if __name__ == "__main__":
main()
+12 -11
View File
@@ -2,7 +2,7 @@ import logging
import os
from twisted.internet import defer
from lbrynet import conf
from lbrynet.conf import Config
from lbrynet.schema.fee import Fee
from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
@@ -31,17 +31,18 @@ log = logging.getLogger(__name__)
class GetStream:
def __init__(self, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder, rate_limiter,
payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None, timeout=None,
reactor=None):
def __init__(self, conf: Config, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder,
rate_limiter, payment_rate_manager, storage, max_key_fee, disable_max_key_fee, data_rate=None,
timeout=None, reactor=None):
if not reactor:
from twisted.internet import reactor
self.conf = conf
self.reactor = reactor
self.timeout = timeout or conf.settings['download_timeout']
self.data_rate = data_rate or conf.settings['data_rate']
self.max_key_fee = max_key_fee or conf.settings['max_key_fee'][1]
self.disable_max_key_fee = disable_max_key_fee or conf.settings['disable_max_key_fee']
self.download_directory = conf.settings.download_dir
self.timeout = timeout or conf.download_timeout
self.data_rate = data_rate or conf.data_rate
self.max_key_fee = max_key_fee or conf.max_key_fee
self.disable_max_key_fee = disable_max_key_fee or conf.disable_max_key_fee
self.download_directory = conf.download_dir
self.timeout_counter = 0
self.code = None
self.sd_hash = None
@@ -153,8 +154,8 @@ class GetStream:
@defer.inlineCallbacks
def _download_sd_blob(self):
sd_blob = yield download_sd_blob(
self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
self.wallet, self.timeout, conf.settings['download_mirrors']
self.conf, self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.payment_rate_manager,
self.wallet, self.timeout, self.conf.download_mirrors
)
defer.returnValue(sd_blob)
+6 -4
View File
@@ -3,27 +3,29 @@ import logging
from twisted.internet import defer, task
from lbrynet.extras.compat import f2d
from lbrynet import utils, conf
from lbrynet import utils
from lbrynet.conf import Config
log = logging.getLogger(__name__)
class DHTHashAnnouncer:
def __init__(self, dht_node, storage, concurrent_announcers=None):
def __init__(self, conf: Config, dht_node, storage, concurrent_announcers=None):
self.conf = conf
self.dht_node = dht_node
self.storage = storage
self.clock = dht_node.clock
self.peer_port = dht_node.peerPort
self.hash_queue = []
if concurrent_announcers is None:
self.concurrent_announcers = conf.settings['concurrent_announcers']
self.concurrent_announcers = conf.concurrent_announcers
else:
self.concurrent_announcers = concurrent_announcers
self._manage_lc = None
if self.concurrent_announcers:
self._manage_lc = task.LoopingCall(self.manage)
self._manage_lc.clock = self.clock
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.settings['concurrent_announcers'] or 1)
self.sem = defer.DeferredSemaphore(self.concurrent_announcers or conf.concurrent_announcers or 1)
def start(self):
if self._manage_lc:
+1 -2
View File
@@ -2,7 +2,6 @@ import binascii
import logging
from twisted.internet import defer
from lbrynet import conf
log = logging.getLogger(__name__)
@@ -59,7 +58,7 @@ class DHTPeerFinder(DummyPeerFinder):
def _execute_peer_search(self, dht_node, blob_hash, timeout):
bin_hash = binascii.unhexlify(blob_hash)
finished_deferred = dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
timeout = timeout or conf.settings['peer_search_timeout']
timeout = timeout or self.component_manager.conf.peer_search_timeout
if timeout:
finished_deferred.addTimeout(timeout, dht_node.clock)
try:
+75 -110
View File
@@ -1,12 +1,12 @@
import asyncio
import collections
import logging
import asyncio
import aiohttp
from lbrynet import conf, utils
from lbrynet import utils
from lbrynet.conf import Config, ANALYTICS_ENDPOINT, ANALYTICS_TOKEN
from lbrynet.extras import system_info
from lbrynet.extras.daemon.storage import looping_call
# Things We Track
SERVER_STARTUP = 'Server Startup'
@@ -28,22 +28,68 @@ log = logging.getLogger(__name__)
class Manager:
def __init__(self, analytics_api, context=None, installation_id=None, session_id=None):
self.analytics_api = analytics_api
def __init__(self, conf: Config, installation_id: str, session_id: str):
self.cookies = {}
self.url = ANALYTICS_ENDPOINT
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
self._enabled = conf.share_usage_data
self._tracked_data = collections.defaultdict(list)
self.looping_tasks = {}
self.context = context or self._make_context(
system_info.get_platform(), conf.settings['wallet'])
self.installation_id = installation_id or conf.settings.installation_id
self.session_id = session_id or conf.settings.get_session_id()
self.is_started = False
self.context = self._make_context(system_info.get_platform(), 'torba')
self.installation_id = installation_id
self.session_id = session_id
self.task: asyncio.Task = None
@classmethod
def new_instance(cls, enabled=None):
api = Api.new_instance(enabled)
return cls(api)
@property
def is_started(self):
return self.task is not None
def start(self):
if self._enabled and self.task is None:
self.task = asyncio.create_task(self.run())
log.info("Start")
async def run(self):
while True:
await self._send_heartbeat()
await asyncio.sleep(1800)
def stop(self):
if self.task is not None and not self.task.done():
self.task.cancel()
async def _post(self, endpoint, data):
# there is an issue with a timing condition with keep-alive
# that is best explained here: https://github.com/mikem23/keepalive-race
#
# If you make a request, wait just the right amount of time,
# then make another request, the requests module may opt to
# reuse the connection, but by the time the server gets it the
# timeout will have expired.
#
# by forcing the connection to close, we will disable the keep-alive.
assert endpoint[0] == '/'
request_kwargs = {
'method': 'POST',
'url': self.url + endpoint,
'headers': {'Connection': 'Close'},
'auth': aiohttp.BasicAuth(self._write_key, ''),
'json': data,
'cookies': self.cookies
}
try:
async with aiohttp.request(**request_kwargs) as response:
self.cookies.update(response.cookies)
except Exception as e:
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
async def track(self, event):
"""Send a single tracking event"""
if self._enabled:
log.debug('Sending track event: %s', event)
await self._post('/track', event)
# Things We Track
async def send_new_download_start(self, download_id, name, claim_dict):
await self._send_new_download_stats("start", download_id, name, claim_dict)
@@ -57,7 +103,7 @@ class Manager:
})
async def _send_new_download_stats(self, action, download_id, name, claim_dict, e=None):
await self.analytics_api.track({
await self.track({
'userId': 'lbry', # required, see https://segment.com/docs/sources/server/http/#track
'event': NEW_DOWNLOAD_STAT,
'properties': self._event_properties({
@@ -72,7 +118,7 @@ class Manager:
})
async def send_upnp_setup_success_fail(self, success, status):
await self.analytics_api.track(
await self.track(
self._event(UPNP_SETUP, {
'success': success,
'status': status,
@@ -80,71 +126,44 @@ class Manager:
)
async def send_server_startup(self):
await self.analytics_api.track(self._event(SERVER_STARTUP))
await self.track(self._event(SERVER_STARTUP))
async def send_server_startup_success(self):
await self.analytics_api.track(self._event(SERVER_STARTUP_SUCCESS))
await self.track(self._event(SERVER_STARTUP_SUCCESS))
async def send_server_startup_error(self, message):
await self.analytics_api.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
await self.track(self._event(SERVER_STARTUP_ERROR, {'message': message}))
async def send_download_started(self, id_, name, claim_dict=None):
await self.analytics_api.track(
await self.track(
self._event(DOWNLOAD_STARTED, self._download_properties(id_, name, claim_dict))
)
async def send_download_errored(self, err, id_, name, claim_dict, report):
download_error_properties = self._download_error_properties(err, id_, name, claim_dict,
report)
await self.analytics_api.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
await self.track(self._event(DOWNLOAD_ERRORED, download_error_properties))
async def send_download_finished(self, id_, name, report, claim_dict=None):
download_properties = self._download_properties(id_, name, claim_dict, report)
await self.analytics_api.track(self._event(DOWNLOAD_FINISHED, download_properties))
await self.track(self._event(DOWNLOAD_FINISHED, download_properties))
async def send_claim_action(self, action):
await self.analytics_api.track(self._event(CLAIM_ACTION, {'action': action}))
await self.track(self._event(CLAIM_ACTION, {'action': action}))
async def send_new_channel(self):
await self.analytics_api.track(self._event(NEW_CHANNEL))
await self.track(self._event(NEW_CHANNEL))
async def send_credits_sent(self):
await self.analytics_api.track(self._event(CREDITS_SENT))
await self.track(self._event(CREDITS_SENT))
async def _send_heartbeat(self):
await self.analytics_api.track(self._event(HEARTBEAT))
await self.track(self._event(HEARTBEAT))
async def _update_tracked_metrics(self):
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
if should_send:
await self.analytics_api.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
# Setup / Shutdown
def start(self):
if not self.is_started:
for name, fn, secs in self._get_looping_calls():
self.looping_tasks[name] = asyncio.create_task(looping_call(secs, fn))
self.is_started = True
log.info("Start")
def shutdown(self):
if self.is_started:
try:
for name, task in self.looping_tasks.items():
if task:
task.cancel()
self.looping_tasks[name] = None
log.info("Stopped analytics looping calls")
self.is_started = False
except Exception as e:
log.exception('Got exception when trying to cancel tasks in analytics: ', exc_info=e)
def _get_looping_calls(self) -> list:
return [
('send_heartbeat', self._send_heartbeat, 300),
('update_tracked_metrics', self._update_tracked_metrics, 600),
]
await self.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
def add_observation(self, metric, value):
self._tracked_data[metric].append(value)
@@ -229,57 +248,3 @@ class Manager:
context['os']['desktop'] = platform['desktop']
context['os']['distro'] = platform['distro']
return context
class Api:
def __init__(self, cookies, url, write_key, enabled):
self.cookies = cookies
self.url = url
self._write_key = write_key
self._enabled = enabled
async def _post(self, endpoint, data):
# there is an issue with a timing condition with keep-alive
# that is best explained here: https://github.com/mikem23/keepalive-race
#
# If you make a request, wait just the right amount of time,
# then make another request, the requests module may opt to
# reuse the connection, but by the time the server gets it the
# timeout will have expired.
#
# by forcing the connection to close, we will disable the keep-alive.
assert endpoint[0] == '/'
request_kwargs = {
'method': 'POST',
'url': self.url + endpoint,
'headers': {'Connection': 'Close'},
'auth': aiohttp.BasicAuth(self._write_key, ''),
'json': data,
'cookies': self.cookies
}
try:
async with aiohttp.request(**request_kwargs) as response:
self.cookies.update(response.cookies)
except Exception as e:
log.exception('Encountered an exception while POSTing to %s: ', self.url + endpoint, exc_info=e)
async def track(self, event):
"""Send a single tracking event"""
if not self._enabled:
return 'Analytics disabled'
log.debug('Sending track event: %s', event)
await self._post('/track', event)
@classmethod
def new_instance(cls, enabled=None):
"""Initialize an instance using values from the configuration"""
if enabled is None:
enabled = conf.settings['share_usage_data']
return cls(
{},
conf.settings['ANALYTICS_ENDPOINT'],
utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']),
enabled,
)
@@ -1,11 +1,9 @@
import asyncio
import aiohttp
import time
import logging
import json
import treq
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from lbrynet.p2p.Error import InvalidExchangeRateResponse, CurrencyConversionError
log = logging.getLogger(__name__)
@@ -38,14 +36,14 @@ class MarketFeed:
REQUESTS_TIMEOUT = 20
EXCHANGE_RATE_UPDATE_RATE_SEC = 300
def __init__(self, market, name, url, params, fee):
def __init__(self, market: str, name: str, url: str, params, fee):
self.market = market
self.name = name
self.url = url
self.params = params
self.fee = fee
self.rate = None
self._updater = LoopingCall(self._update_price)
self._task: asyncio.Task = None
self._online = True
def rate_is_initialized(self):
@@ -54,17 +52,16 @@ class MarketFeed:
def is_online(self):
return self._online
@defer.inlineCallbacks
def _make_request(self):
response = yield treq.get(self.url, params=self.params, timeout=self.REQUESTS_TIMEOUT)
defer.returnValue((yield response.content()))
async def _make_request(self):
async with aiohttp.request('get', self.url, params=self.params) as response:
return (await response.read()).decode()
def _handle_response(self, response):
return NotImplementedError
raise NotImplementedError()
def _subtract_fee(self, from_amount):
# increase amount to account for market fees
return defer.succeed(from_amount / (1.0 - self.fee))
return from_amount / (1.0 - self.fee)
def _save_price(self, price):
log.debug("Saving price update %f for %s from %s" % (price, self.market, self.name))
@@ -77,21 +74,23 @@ class MarketFeed:
log.debug("Exchange rate error (%s from %s): %s", self.market, self.name, err)
self._online = False
def _update_price(self):
d = self._make_request()
d.addCallback(self._handle_response)
d.addCallback(self._subtract_fee)
d.addCallback(self._save_price)
d.addErrback(self._on_error)
return d
async def _update_price(self):
while True:
try:
response = await asyncio.wait_for(self._make_request(), self.REQUESTS_TIMEOUT)
self._save_price(self._subtract_fee(self._handle_response(response)))
except (asyncio.TimeoutError, InvalidExchangeRateResponse) as err:
self._on_error(err)
await asyncio.sleep(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
def start(self):
if not self._updater.running:
self._updater.start(self.EXCHANGE_RATE_UPDATE_RATE_SEC)
if not self._task:
self._task = asyncio.create_task(self._update_price())
def stop(self):
if self._updater.running:
self._updater.stop()
if self._task and not self._task.done():
self._task.cancel()
self._task = None
class BittrexFeed(MarketFeed):
@@ -116,7 +115,7 @@ class BittrexFeed(MarketFeed):
if totals <= 0 or qtys <= 0:
raise InvalidExchangeRateResponse(self.market, 'quantities were not positive')
vwap = totals / qtys
return defer.succeed(float(1.0 / vwap))
return float(1.0 / vwap)
class LBRYioFeed(MarketFeed):
@@ -133,7 +132,7 @@ class LBRYioFeed(MarketFeed):
json_response = json.loads(response)
if 'data' not in json_response:
raise InvalidExchangeRateResponse(self.name, 'result not found')
return defer.succeed(1.0 / json_response['data']['lbc_btc'])
return 1.0 / json_response['data']['lbc_btc']
class LBRYioBTCFeed(MarketFeed):
@@ -153,7 +152,7 @@ class LBRYioBTCFeed(MarketFeed):
raise InvalidExchangeRateResponse(self.name, "invalid rate response : %s" % response)
if 'data' not in json_response:
raise InvalidExchangeRateResponse(self.name, 'result not found')
return defer.succeed(1.0 / json_response['data']['btc_usd'])
return 1.0 / json_response['data']['btc_usd']
class CryptonatorBTCFeed(MarketFeed):
@@ -174,7 +173,7 @@ class CryptonatorBTCFeed(MarketFeed):
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
'success' not in json_response or json_response['success'] is not True:
raise InvalidExchangeRateResponse(self.name, 'result not found')
return defer.succeed(float(json_response['ticker']['price']))
return float(json_response['ticker']['price'])
class CryptonatorFeed(MarketFeed):
@@ -195,7 +194,7 @@ class CryptonatorFeed(MarketFeed):
if 'ticker' not in json_response or len(json_response['ticker']) == 0 or \
'success' not in json_response or json_response['success'] is not True:
raise InvalidExchangeRateResponse(self.name, 'result not found')
return defer.succeed(float(json_response['ticker']['price']))
return float(json_response['ticker']['price'])
class ExchangeRateManager:
+64
View File
@@ -0,0 +1,64 @@
import asyncio
import aiohttp
import json
import logging.handlers
import traceback
from lbrynet import utils, __version__
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
class JsonFormatter(logging.Formatter):
"""Format log records using json serialization"""
def __init__(self, **kwargs):
self.attributes = kwargs
def format(self, record):
data = {
'loggerName': record.name,
'asciTime': self.formatTime(record),
'fileName': record.filename,
'functionName': record.funcName,
'levelNo': record.levelno,
'lineNo': record.lineno,
'levelName': record.levelname,
'message': record.getMessage(),
}
data.update(self.attributes)
if record.exc_info:
data['exc_info'] = self.formatException(record.exc_info)
return json.dumps(data)
class HTTPSLogglyHandler(logging.Handler):
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
super().__init__()
self.fqdn = fqdn
self.localname = localname
self.facility = facility
self.cookies = cookies or {}
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
)
def get_full_message(self, record):
if record.exc_info:
return '\n'.join(traceback.format_exception(*record.exc_info))
else:
return record.getMessage()
async def _emit(self, record):
payload = self.format(record)
async with aiohttp.request('post', self.url, data=payload.encode(), cookies=self.cookies) as response:
self.cookies.update(response.cookies)
def emit(self, record):
asyncio.ensure_future(self._emit(record))
def get_loggly_handler(loggly_token=LOGGLY_TOKEN):
handler = HTTPSLogglyHandler(loggly_token)
handler.setFormatter(JsonFormatter())
return handler
+2 -2
View File
@@ -3,7 +3,7 @@ import logging
log = logging.getLogger(__name__)
def migrate_db(db_dir, start, end):
def migrate_db(conf, start, end):
current = start
while current < end:
if current == 1:
@@ -25,7 +25,7 @@ def migrate_db(db_dir, start, end):
else:
raise Exception("DB migration of version {} to {} is not available".format(current,
current+1))
do_migration(db_dir)
do_migration(conf)
current += 1
log.info("successfully migrated the database from revision %i to %i", current - 1, current)
return None
@@ -5,9 +5,9 @@ import logging
log = logging.getLogger(__name__)
UNSET_NOUT = -1
def do_migration(db_dir):
def do_migration(conf):
log.info("Doing the migration")
migrate_blockchainname_db(db_dir)
migrate_blockchainname_db(conf.data_dir)
log.info("Migration succeeded")
@@ -5,9 +5,9 @@ import logging
log = logging.getLogger(__name__)
def do_migration(db_dir):
def do_migration(conf):
log.info("Doing the migration")
migrate_blockchainname_db(db_dir)
migrate_blockchainname_db(conf.data_dir)
log.info("Migration succeeded")
@@ -5,9 +5,9 @@ import logging
log = logging.getLogger(__name__)
def do_migration(db_dir):
def do_migration(conf):
log.info("Doing the migration")
migrate_blobs_db(db_dir)
migrate_blobs_db(conf.data_dir)
log.info("Migration succeeded")
@@ -5,9 +5,9 @@ import logging
log = logging.getLogger(__name__)
def do_migration(db_dir):
def do_migration(conf):
log.info("Doing the migration")
add_lbry_file_metadata(db_dir)
add_lbry_file_metadata(conf.data_dir)
log.info("Migration succeeded")
@@ -3,7 +3,6 @@ import os
import json
import logging
from binascii import hexlify
from lbrynet import conf
from lbrynet.schema.decode import smart_decode
log = logging.getLogger(__name__)
@@ -104,13 +103,13 @@ def verify_sd_blob(sd_hash, blob_dir):
return decoded, sd_length
def do_migration(db_dir):
new_db_path = os.path.join(db_dir, "lbrynet.sqlite")
def do_migration(conf):
new_db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(new_db_path)
metadata_db = sqlite3.connect(os.path.join(db_dir, "blockchainname.db"))
lbryfile_db = sqlite3.connect(os.path.join(db_dir, 'lbryfile_info.db'))
blobs_db = sqlite3.connect(os.path.join(db_dir, 'blobs.db'))
metadata_db = sqlite3.connect(os.path.join(conf.data_dir, "blockchainname.db"))
lbryfile_db = sqlite3.connect(os.path.join(conf.data_dir, 'lbryfile_info.db'))
blobs_db = sqlite3.connect(os.path.join(conf.data_dir, 'blobs.db'))
name_metadata_cursor = metadata_db.cursor()
lbryfile_cursor = lbryfile_db.cursor()
@@ -186,7 +185,7 @@ def do_migration(db_dir):
(stream_hash, blob_hash, position, iv)
)
download_dir = conf.settings.download_dir
download_dir = conf.download_dir
if not isinstance(download_dir, bytes):
download_dir = download_dir.encode()
@@ -278,7 +277,7 @@ def do_migration(db_dir):
# recover damaged streams
if damaged_stream_sds:
blob_dir = os.path.join(db_dir, "blobfiles")
blob_dir = os.path.join(conf.data_dir, "blobfiles")
damaged_sds_on_disk = [] if not os.path.isdir(blob_dir) else list({p for p in os.listdir(blob_dir)
if p in damaged_stream_sds})
for damaged_sd in damaged_sds_on_disk:
@@ -316,7 +315,7 @@ def do_migration(db_dir):
log.warning("detected a failed previous migration to revision 6, repairing it")
connection.close()
os.remove(new_db_path)
return do_migration(db_dir)
return do_migration(conf)
raise err
connection.close()
@@ -2,8 +2,8 @@ import sqlite3
import os
def do_migration(db_dir):
db_path = os.path.join(db_dir, "lbrynet.sqlite")
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
cursor.executescript("alter table blob add last_announced_time integer;")
@@ -2,8 +2,8 @@ import sqlite3
import os
def do_migration(db_dir):
db_path = os.path.join(db_dir, "lbrynet.sqlite")
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
@@ -9,9 +9,9 @@ from lbrynet.blob.CryptBlob import CryptBlobInfo
log = logging.getLogger(__name__)
def do_migration(db_dir):
db_path = os.path.join(db_dir, "lbrynet.sqlite")
blob_dir = os.path.join(db_dir, "blobfiles")
def do_migration(conf):
db_path = os.path.join(conf.data_dir, "lbrynet.sqlite")
blob_dir = os.path.join(conf.data_dir, "blobfiles")
connection = sqlite3.connect(db_path)
cursor = connection.cursor()
+6 -5
View File
@@ -5,7 +5,7 @@ import traceback
import typing
from binascii import hexlify, unhexlify
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
from lbrynet import conf
from lbrynet.conf import Config
from lbrynet.schema.claim import ClaimDict
from lbrynet.schema.decode import smart_decode
from lbrynet.blob.CryptBlob import CryptBlobInfo
@@ -132,15 +132,16 @@ class SQLiteStorage(SQLiteMixin):
);
"""
def __init__(self, path, loop=None):
def __init__(self, conf: Config, path, loop=None):
super().__init__(path)
self.conf = conf
self.content_claim_callbacks = {}
self.check_should_announce_lc = None
self.loop = loop or asyncio.get_event_loop()
async def open(self):
await super().open()
if 'reflector' not in conf.settings['components_to_skip']:
if 'reflector' not in self.conf.components_to_skip:
self.check_should_announce_lc = looping_call(
600, self.verify_will_announce_all_head_and_sd_blobs
)
@@ -236,7 +237,7 @@ class SQLiteStorage(SQLiteMixin):
def get_blobs_to_announce(self):
def get_and_update(transaction):
timestamp = self.loop.time()
if conf.settings['announce_head_blobs_only']:
if self.conf.announce_head_blobs_only:
r = transaction.execute(
"select blob_hash from blob "
"where blob_hash is not null and "
@@ -797,7 +798,7 @@ class SQLiteStorage(SQLiteMixin):
"select s.sd_hash from stream s "
"left outer join reflected_stream r on s.sd_hash=r.sd_hash "
"where r.timestamp is null or r.timestamp < ?",
self.loop.time() - conf.settings['auto_re_reflect_interval']
self.loop.time() - self.conf.auto_re_reflect_interval
)
+10 -34
View File
@@ -1,7 +1,4 @@
import random
from twisted.internet import reactor, defer
from lbrynet import conf
from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory
from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory
@@ -48,40 +45,19 @@ def _reflect_blobs(blob_manager, blob_hashes, reflector_server):
return result
def reflect_file(lbry_file, reflector_server=None):
if reflector_server:
if len(reflector_server.split(":")) == 2:
host, port = tuple(reflector_server.split(":"))
reflector_server = host, int(port)
else:
reflector_server = reflector_server, 5566
def reflect_file(lbry_file, reflector_server):
if len(reflector_server.split(":")) == 2:
host, port = tuple(reflector_server.split(":"))
reflector_server = host, int(port)
else:
reflector_server = random.choice(conf.settings['reflector_servers'])
reflector_server = reflector_server, 5566
return _reflect_file(lbry_file, reflector_server)
@defer.inlineCallbacks
def reflect_stream(blob_manager, stream_hash, reflector_server=None):
if reflector_server:
if len(reflector_server.split(":")) == 2:
host, port = tuple(reflector_server.split(":"))
reflector_server = host, int(port)
else:
reflector_server = reflector_server, 5566
def reflect_blob_hashes(blob_hashes, blob_manager, reflector_server):
if len(reflector_server.split(":")) == 2:
host, port = tuple(reflector_server.split(":"))
reflector_server = host, int(port)
else:
reflector_server = random.choice(conf.settings['reflector_servers'])
sd_hash = yield blob_manager.storage.get_sd_blob_hash_for_stream(stream_hash)
result = yield _reflect_stream(blob_manager, stream_hash, sd_hash, reflector_server)
defer.returnValue(result)
def reflect_blob_hashes(blob_hashes, blob_manager, reflector_server=None):
if reflector_server:
if len(reflector_server.split(":")) == 2:
host, port = tuple(reflector_server.split(":"))
reflector_server = host, int(port)
else:
reflector_server = reflector_server, 5566
else:
reflector_server = random.choice(conf.settings['reflector_servers'])
reflector_server = reflector_server, 5566
return _reflect_blobs(blob_manager, blob_hashes, reflector_server)
+3 -6
View File
@@ -166,14 +166,12 @@ class LbryWalletManager(BaseWalletManager):
'lbrycrd_main': 'lbc_mainnet',
'lbrycrd_testnet': 'lbc_testnet',
'lbrycrd_regtest': 'lbc_regtest'
}[settings['blockchain_name']]
}[settings.blockchain_name]
ledger_config = {
'auto_connect': True,
'default_servers': settings['lbryum_servers'],
'default_servers': settings.lbryum_servers,
'data_path': settings.wallet_dir,
'use_keyring': settings['use_keyring'],
#'db': db
}
wallets_directory = os.path.join(settings.wallet_dir, 'wallets')
@@ -413,8 +411,7 @@ class LbryWalletManager(BaseWalletManager):
claim_address = await account.receiving.get_or_create_usable_address()
if certificate:
claim = claim.sign(
certificate.private_key, claim_address, certificate.claim_id, curve=SECP256k1, name=name,
force_detached=False # TODO: delete it and make True default everywhere when its out
certificate.private_key, claim_address, certificate.claim_id, curve=SECP256k1, name=name
)
existing_claims = await account.get_claims(
claim_name_type__any={'is_claim': 1, 'is_update': 1}, # exclude is_supports
-89
View File
@@ -1,43 +1,11 @@
import json
import logging
import logging.handlers
import sys
import traceback
import treq
import twisted.python.log
from twisted.internet import defer
from lbrynet import __version__ as lbrynet_version, build_type
from lbrynet import utils, conf
class HTTPSHandler(logging.Handler):
def __init__(self, url, fqdn=False, localname=None, facility=None, cookies=None):
super().__init__()
self.url = url
self.fqdn = fqdn
self.localname = localname
self.facility = facility
self.cookies = cookies or {}
def get_full_message(self, record):
if record.exc_info:
return '\n'.join(traceback.format_exception(*record.exc_info))
else:
return record.getMessage()
@defer.inlineCallbacks
def _emit(self, record):
payload = self.format(record)
response = yield treq.post(self.url, data=payload.encode(), cookies=self.cookies)
self.cookies.update(response.cookies())
def emit(self, record):
return self._emit(record)
DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s"
DEFAULT_FORMATTER = logging.Formatter(DEFAULT_FORMAT)
LOGGLY_URL = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}"
def remove_handlers(log, handler_name):
@@ -108,63 +76,6 @@ def configure_file_handler(file_name, **kwargs):
return handler
def get_loggly_url(token=None, version=None):
token = token or utils.deobfuscate(conf.settings['LOGGLY_TOKEN'])
version = version or lbrynet_version
return LOGGLY_URL.format(token=token, tag='lbrynet-' + version)
def configure_loggly_handler():
if build_type.BUILD == 'dev':
return
if not conf.settings['share_usage_data']:
return
level = logging.ERROR
handler = get_loggly_handler(level=level, installation_id=conf.settings.installation_id,
session_id=conf.settings.get_session_id())
log = logging.getLogger("lbrynet")
if handler.name:
remove_handlers(log, handler.name)
handler.setLevel(level)
log.addHandler(handler)
# need to reduce the logger's level down to the
# handler's level or else the handler won't
# get those messages
if log.level > level:
log.setLevel(level)
def get_loggly_handler(level, installation_id, session_id):
formatter = JsonFormatter(level=level, installation_id=installation_id, session_id=session_id)
handler = HTTPSHandler(get_loggly_url())
handler.setFormatter(formatter)
handler.name = 'loggly'
return handler
class JsonFormatter(logging.Formatter):
"""Format log records using json serialization"""
def __init__(self, **kwargs):
self.attributes = kwargs
def format(self, record):
data = {
'loggerName': record.name,
'asciTime': self.formatTime(record),
'fileName': record.filename,
'functionName': record.funcName,
'levelNo': record.levelno,
'lineNo': record.lineno,
'levelName': record.levelname,
'message': record.getMessage(),
}
data.update(self.attributes)
if record.exc_info:
data['exc_info'] = self.formatException(record.exc_info)
return json.dumps(data)
def failure(failure, log, msg, *args):
"""Log a failure message from a deferred.
+1 -1
View File
@@ -157,7 +157,7 @@ class HTTPBlobDownloader:
@defer.inlineCallbacks
def _write_blob(self, writer, blob):
response = yield self.client.get(url_for(choice(self.servers), blob.blob_hash))
response = yield self.client.get(url_for('{}:{}'.format(*choice(self.servers)), blob.blob_hash))
if response.code != 200:
log.debug('Missing a blob: %s', blob.blob_hash)
if blob.blob_hash in self.blob_hashes:
+8 -10
View File
@@ -1,13 +1,11 @@
from decimal import Decimal
from lbrynet import conf
from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy
class BasePaymentRateManager:
def __init__(self, rate=None, info_rate=None):
self.min_blob_data_payment_rate = rate if rate is not None else conf.settings['data_rate']
self.min_blob_info_payment_rate = (
info_rate if info_rate is not None else conf.settings['min_info_rate'])
def __init__(self, rate, info_rate):
self.min_blob_data_payment_rate = rate
self.min_blob_info_payment_rate = info_rate
class PaymentRateManager:
@@ -37,7 +35,7 @@ class PaymentRateManager:
class NegotiatedPaymentRateManager:
def __init__(self, base, availability_tracker, generous=None):
def __init__(self, base, availability_tracker, generous):
"""
@param base: a BasePaymentRateManager
@param availability_tracker: a BlobAvailabilityTracker
@@ -48,10 +46,10 @@ class NegotiatedPaymentRateManager:
self.min_blob_data_payment_rate = base.min_blob_data_payment_rate
self.points_paid = 0.0
self.blob_tracker = availability_tracker
self.generous = generous if generous is not None else conf.settings['is_generous_host']
self.strategy = get_default_strategy(self.blob_tracker,
base_price=self.base.min_blob_data_payment_rate,
is_generous=generous)
self.generous = generous
self.strategy = get_default_strategy(
self.blob_tracker, self.base.min_blob_data_payment_rate, generous
)
def get_rate_blob_data(self, peer, blobs):
response = self.strategy.make_offer(peer, blobs)
+1 -3
View File
@@ -1,5 +1,4 @@
from decimal import Decimal
from lbrynet import conf
def get_default_price_model(blob_tracker, base_price, **kwargs):
@@ -26,9 +25,8 @@ class MeanAvailabilityWeightedPrice:
"""
def __init__(self, tracker, base_price=None, alpha=1.0):
def __init__(self, tracker, base_price, alpha=1.0):
self.blob_tracker = tracker
base_price = base_price if base_price is not None else conf.settings['data_rate']
self.base_price = Decimal(base_price)
self.alpha = Decimal(alpha)
+5 -3
View File
@@ -14,6 +14,7 @@ from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder
from lbrynet.conf import Config
log = logging.getLogger(__name__)
@@ -60,7 +61,8 @@ class SingleBlobDownloadManager:
class SinglePeerDownloader:
def __init__(self):
def __init__(self, conf: Config):
self.conf = conf
self._payment_rate_manager = OnlyFreePaymentsManager()
self._rate_limiter = DummyRateLimiter()
self._wallet = None
@@ -81,7 +83,7 @@ class SinglePeerDownloader:
peer_finder = SinglePeerFinder(peer)
requester = BlobRequester(blob_manager, peer_finder, self._payment_rate_manager,
self._wallet, download_manager)
downloader = StandaloneBlobDownloader(blob_hash, blob_manager, peer_finder,
downloader = StandaloneBlobDownloader(self.conf, blob_hash, blob_manager, peer_finder,
self._rate_limiter, self._payment_rate_manager,
self._wallet, timeout=timeout)
info_exchanger = self._wallet.get_info_exchanger()
@@ -96,7 +98,7 @@ class SinglePeerDownloader:
defer.returnValue(result)
async def download_temp_blob_from_peer(self, peer, timeout, blob_hash):
tmp_storage = SQLiteStorage(':memory:')
tmp_storage = SQLiteStorage(Config(), ':memory:')
await tmp_storage.open()
tmp_dir = tempfile.mkdtemp()
tmp_blob_manager = DiskBlobManager(tmp_dir, tmp_storage)
+8 -12
View File
@@ -1,11 +1,10 @@
from decimal import Decimal
from lbrynet import conf
from lbrynet.p2p.Offer import Offer
from lbrynet.p2p.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
def get_default_strategy(blob_tracker, **kwargs):
return BasicAvailabilityWeightedStrategy(blob_tracker, **kwargs)
def get_default_strategy(blob_tracker, base_price, is_generous, **kwargs):
return BasicAvailabilityWeightedStrategy(blob_tracker, base_price, is_generous, **kwargs)
class Strategy:
@@ -13,10 +12,9 @@ class Strategy:
Base for negotiation strategies
"""
def __init__(self, price_model, max_rate, min_rate, is_generous=None):
def __init__(self, price_model, max_rate, min_rate, is_generous):
self.price_model = price_model
self.is_generous = (
is_generous if is_generous is not None else conf.settings['is_generous_host'])
self.is_generous = is_generous
self.accepted_offers = {}
self.pending_sent_offers = {}
self.offers_sent = {}
@@ -98,13 +96,11 @@ class BasicAvailabilityWeightedStrategy(Strategy):
"""
def __init__(self, blob_tracker, acceleration=1.25,
deceleration=0.9, max_rate=None,
min_rate=0.0,
is_generous=None,
base_price=0.0001, alpha=1.0):
def __init__(self, blob_tracker, base_price, is_generous,
acceleration=1.25, deceleration=0.9, max_rate=None,
min_rate=0.0, alpha=1.0):
price_model = MeanAvailabilityWeightedPrice(
blob_tracker, base_price=base_price, alpha=alpha)
blob_tracker, base_price, alpha=alpha)
super().__init__(price_model, max_rate, min_rate, is_generous)
self._acceleration = Decimal(acceleration) # rate of how quickly to ramp offer
self._deceleration = Decimal(deceleration)
+4 -3
View File
@@ -429,8 +429,8 @@ class EncryptedFileStreamDescriptorValidator:
@defer.inlineCallbacks
def download_sd_blob(blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet, timeout=None,
download_mirrors=None):
def download_sd_blob(conf, blob_hash, blob_manager, peer_finder, rate_limiter, payment_rate_manager, wallet,
timeout=None, download_mirrors=None):
"""
Downloads a single blob from the network
@@ -443,7 +443,8 @@ def download_sd_blob(blob_hash, blob_manager, peer_finder, rate_limiter, payment
@return: An object of type HashBlob
"""
downloader = StandaloneBlobDownloader(blob_hash,
downloader = StandaloneBlobDownloader(conf,
blob_hash,
blob_manager,
peer_finder,
rate_limiter,
+3 -2
View File
@@ -5,7 +5,8 @@ from twisted.internet import error, defer
from twisted.internet.protocol import Protocol, ClientFactory
from twisted.protocols.policies import TimeoutMixin
from twisted.python import failure
from lbrynet import conf, utils
from lbrynet import utils
from lbrynet.conf import MAX_RESPONSE_INFO_SIZE
from lbrynet.p2p.Error import ConnectionClosedBeforeResponseError, NoResponseError
from lbrynet.p2p.Error import DownloadCanceledError, MisbehavingPeerError
from lbrynet.p2p.Error import RequestCanceledError
@@ -52,7 +53,7 @@ class ClientProtocol(Protocol, TimeoutMixin):
self._blob_download_request.write(data)
else:
self._response_buff += data
if len(self._response_buff) > conf.settings['MAX_RESPONSE_INFO_SIZE']:
if len(self._response_buff) > MAX_RESPONSE_INFO_SIZE:
log.warning("Response is too large from %s. Size %s",
self.peer, len(self._response_buff))
self.transport.loseConnection()
+6 -5
View File
@@ -1,7 +1,8 @@
import random
import logging
from twisted.internet import defer, reactor
from lbrynet import utils, conf
from lbrynet import utils
from lbrynet.conf import Config
from lbrynet.p2p.client.ClientProtocol import ClientProtocolFactory
from lbrynet.p2p.Error import InsufficientFundsError
@@ -20,11 +21,11 @@ class ConnectionManager:
MANAGE_CALL_INTERVAL_SEC = 5
TCP_CONNECT_TIMEOUT = 15
def __init__(self, downloader, rate_limiter,
primary_request_creators, secondary_request_creators):
def __init__(self, downloader, rate_limiter, primary_request_creators, secondary_request_creators):
self.seek_head_blob_first = conf.settings['seek_head_blob_first']
self.max_connections_per_stream = conf.settings['max_connections_per_stream']
self.conf: Config = downloader.conf
self.seek_head_blob_first = self.conf.seek_head_blob_first
self.max_connections_per_stream = self.conf.max_connections_per_stream
self.downloader = downloader
self.rate_limiter = rate_limiter
@@ -1,4 +1,5 @@
import logging
from lbrynet.conf import Config
from lbrynet.p2p.BlobInfo import BlobInfo
from lbrynet.p2p.client.BlobRequester import BlobRequester
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
@@ -82,9 +83,10 @@ class DummyBlobHandler:
class StandaloneBlobDownloader:
def __init__(self, blob_hash, blob_manager, peer_finder,
def __init__(self, conf: Config, blob_hash, blob_manager, peer_finder,
rate_limiter, payment_rate_manager, wallet,
timeout=None):
self.conf = conf
self.blob_hash = blob_hash
self.blob_manager = blob_manager
self.peer_finder = peer_finder
+2 -2
View File
@@ -162,9 +162,9 @@ class ClaimDict(OrderedDict):
signer = get_signer(curve).load_pem(private_key)
return cls.load_protobuf(signer.certificate)
def sign(self, private_key, claim_address, cert_claim_id, curve=SECP256k1, name=None, force_detached=False):
def sign(self, private_key, claim_address, cert_claim_id, curve=SECP256k1, name=None, legacy=False):
signer = get_signer(curve).load_pem(private_key)
signed, signature = signer.sign_stream_claim(self, claim_address, cert_claim_id, name, force_detached)
signed, signature = signer.sign_stream_claim(self, claim_address, cert_claim_id, name, legacy)
return ClaimDict.load_protobuf(signed, signature)
def validate_signature(self, claim_address, certificate, name=None):
+4
View File
@@ -14,6 +14,10 @@ def encode_fields(claim_dictionary, detached_signature: Signature):
claim_value = claim_dictionary[claim_type]
if claim_type == CLAIM_TYPES[STREAM_TYPE]:
claim_value['source']['source'] = binascii.hexlify(claim_value['source']['source']).decode()
if 'releaseTime' in claim_value['metadata']:
release_time = int(claim_value['metadata'].pop('releaseTime'))
if release_time != 0:
claim_value['metadata']['releaseTime'] = release_time
if 'fee' in claim_value['metadata']:
try:
address = encode_address(claim_value['metadata']['fee']['address'])
-18
View File
@@ -1,18 +0,0 @@
syntax = "proto2";
enum KeyType {
UNKNOWN_PUBLIC_KEY_TYPE = 0;
NIST256p = 1;
NIST384p = 2;
SECP256k1 = 3;
}
message Certificate {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
required Version version = 1;
required KeyType keyType = 2;
required bytes publicKey = 4;
}
+18 -18
View File
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/certificate.proto
# source: certificate.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -16,16 +16,16 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/certificate.proto',
package='',
name='certificate.proto',
package='pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n&lbrynet/schema/proto/certificate.proto\"\x8e\x01\n\x0b\x43\x65rtificate\x12%\n\x07version\x18\x01 \x02(\x0e\x32\x14.Certificate.Version\x12\x19\n\x07keyType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
serialized_pb=_b('\n\x11\x63\x65rtificate.proto\x12\x02pb\"\x94\x01\n\x0b\x43\x65rtificate\x12(\n\x07version\x18\x01 \x02(\x0e\x32\x17.pb.Certificate.Version\x12\x1c\n\x07keyType\x18\x02 \x02(\x0e\x32\x0b.pb.KeyType\x12\x11\n\tpublicKey\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01*Q\n\x07KeyType\x12\x1b\n\x17UNKNOWN_PUBLIC_KEY_TYPE\x10\x00\x12\x0c\n\x08NIST256p\x10\x01\x12\x0c\n\x08NIST384p\x10\x02\x12\r\n\tSECP256k1\x10\x03')
)
_KEYTYPE = _descriptor.EnumDescriptor(
name='KeyType',
full_name='KeyType',
full_name='pb.KeyType',
filename=None,
file=DESCRIPTOR,
values=[
@@ -48,8 +48,8 @@ _KEYTYPE = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=187,
serialized_end=268,
serialized_start=176,
serialized_end=257,
)
_sym_db.RegisterEnumDescriptor(_KEYTYPE)
@@ -62,7 +62,7 @@ SECP256k1 = 3
_CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Certificate.Version',
full_name='pb.Certificate.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -77,35 +77,35 @@ _CERTIFICATE_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=143,
serialized_end=185,
serialized_start=132,
serialized_end=174,
)
_sym_db.RegisterEnumDescriptor(_CERTIFICATE_VERSION)
_CERTIFICATE = _descriptor.Descriptor(
name='Certificate',
full_name='Certificate',
full_name='pb.Certificate',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Certificate.version', index=0,
name='version', full_name='pb.Certificate.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='keyType', full_name='Certificate.keyType', index=1,
name='keyType', full_name='pb.Certificate.keyType', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='publicKey', full_name='Certificate.publicKey', index=2,
name='publicKey', full_name='pb.Certificate.publicKey', index=2,
number=4, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
@@ -124,8 +124,8 @@ _CERTIFICATE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=43,
serialized_end=185,
serialized_start=26,
serialized_end=174,
)
_CERTIFICATE.fields_by_name['version'].enum_type = _CERTIFICATE_VERSION
@@ -137,8 +137,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
Certificate = _reflection.GeneratedProtocolMessageType('Certificate', (_message.Message,), dict(
DESCRIPTOR = _CERTIFICATE,
__module__ = 'lbrynet.schema.proto.certificate_pb2'
# @@protoc_insertion_point(class_scope:Certificate)
__module__ = 'certificate_pb2'
# @@protoc_insertion_point(class_scope:pb.Certificate)
))
_sym_db.RegisterMessage(Certificate)
-22
View File
@@ -1,22 +0,0 @@
syntax = "proto2";
import "lbrynet/schema/proto/stream.proto";
import "lbrynet/schema/proto/certificate.proto";
import "lbrynet/schema/proto/signature.proto";
message Claim {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
required Version version = 1;
enum ClaimType {
UNKNOWN_CLAIM_TYPE = 0;
streamType = 1;
certificateType = 2;
}
required ClaimType claimType = 2;
optional Stream stream = 3;
optional Certificate certificate = 4;
optional Signature publisherSignature = 5;
}
+27 -27
View File
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/claim.proto
# source: claim.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -12,25 +12,25 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
from lbrynet.schema.proto import stream_pb2 as lbrynet_dot_schema_dot_proto_dot_stream__pb2
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
from lbrynet.schema.proto import signature_pb2 as lbrynet_dot_schema_dot_proto_dot_signature__pb2
from . import stream_pb2 as stream__pb2
from . import certificate_pb2 as certificate__pb2
from . import signature_pb2 as signature__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/claim.proto',
package='',
name='claim.proto',
package='pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n lbrynet/schema/proto/claim.proto\x1a!lbrynet/schema/proto/stream.proto\x1a&lbrynet/schema/proto/certificate.proto\x1a$lbrynet/schema/proto/signature.proto\"\xa7\x02\n\x05\x43laim\x12\x1f\n\x07version\x18\x01 \x02(\x0e\x32\x0e.Claim.Version\x12#\n\tclaimType\x18\x02 \x02(\x0e\x32\x10.Claim.ClaimType\x12\x17\n\x06stream\x18\x03 \x01(\x0b\x32\x07.Stream\x12!\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x0c.Certificate\x12&\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\n.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
serialized_pb=_b('\n\x0b\x63laim.proto\x12\x02pb\x1a\x0cstream.proto\x1a\x11\x63\x65rtificate.proto\x1a\x0fsignature.proto\"\xb6\x02\n\x05\x43laim\x12\"\n\x07version\x18\x01 \x02(\x0e\x32\x11.pb.Claim.Version\x12&\n\tclaimType\x18\x02 \x02(\x0e\x32\x13.pb.Claim.ClaimType\x12\x1a\n\x06stream\x18\x03 \x01(\x0b\x32\n.pb.Stream\x12$\n\x0b\x63\x65rtificate\x18\x04 \x01(\x0b\x32\x0f.pb.Certificate\x12)\n\x12publisherSignature\x18\x05 \x01(\x0b\x32\r.pb.Signature\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"H\n\tClaimType\x12\x16\n\x12UNKNOWN_CLAIM_TYPE\x10\x00\x12\x0e\n\nstreamType\x10\x01\x12\x13\n\x0f\x63\x65rtificateType\x10\x02')
,
dependencies=[lbrynet_dot_schema_dot_proto_dot_stream__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_signature__pb2.DESCRIPTOR,])
dependencies=[stream__pb2.DESCRIPTOR,certificate__pb2.DESCRIPTOR,signature__pb2.DESCRIPTOR,])
_CLAIM_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Claim.Version',
full_name='pb.Claim.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -45,14 +45,14 @@ _CLAIM_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=329,
serialized_end=371,
serialized_start=264,
serialized_end=306,
)
_sym_db.RegisterEnumDescriptor(_CLAIM_VERSION)
_CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
name='ClaimType',
full_name='Claim.ClaimType',
full_name='pb.Claim.ClaimType',
filename=None,
file=DESCRIPTOR,
values=[
@@ -71,49 +71,49 @@ _CLAIM_CLAIMTYPE = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=373,
serialized_end=445,
serialized_start=308,
serialized_end=380,
)
_sym_db.RegisterEnumDescriptor(_CLAIM_CLAIMTYPE)
_CLAIM = _descriptor.Descriptor(
name='Claim',
full_name='Claim',
full_name='pb.Claim',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Claim.version', index=0,
name='version', full_name='pb.Claim.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='claimType', full_name='Claim.claimType', index=1,
name='claimType', full_name='pb.Claim.claimType', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='stream', full_name='Claim.stream', index=2,
name='stream', full_name='pb.Claim.stream', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='certificate', full_name='Claim.certificate', index=3,
name='certificate', full_name='pb.Claim.certificate', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='publisherSignature', full_name='Claim.publisherSignature', index=4,
name='publisherSignature', full_name='pb.Claim.publisherSignature', index=4,
number=5, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
@@ -133,15 +133,15 @@ _CLAIM = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=150,
serialized_end=445,
serialized_start=70,
serialized_end=380,
)
_CLAIM.fields_by_name['version'].enum_type = _CLAIM_VERSION
_CLAIM.fields_by_name['claimType'].enum_type = _CLAIM_CLAIMTYPE
_CLAIM.fields_by_name['stream'].message_type = lbrynet_dot_schema_dot_proto_dot_stream__pb2._STREAM
_CLAIM.fields_by_name['certificate'].message_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._CERTIFICATE
_CLAIM.fields_by_name['publisherSignature'].message_type = lbrynet_dot_schema_dot_proto_dot_signature__pb2._SIGNATURE
_CLAIM.fields_by_name['stream'].message_type = stream__pb2._STREAM
_CLAIM.fields_by_name['certificate'].message_type = certificate__pb2._CERTIFICATE
_CLAIM.fields_by_name['publisherSignature'].message_type = signature__pb2._SIGNATURE
_CLAIM_VERSION.containing_type = _CLAIM
_CLAIM_CLAIMTYPE.containing_type = _CLAIM
DESCRIPTOR.message_types_by_name['Claim'] = _CLAIM
@@ -149,8 +149,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
Claim = _reflection.GeneratedProtocolMessageType('Claim', (_message.Message,), dict(
DESCRIPTOR = _CLAIM,
__module__ = 'lbrynet.schema.proto.claim_pb2'
# @@protoc_insertion_point(class_scope:Claim)
__module__ = 'claim_pb2'
# @@protoc_insertion_point(class_scope:pb.Claim)
))
_sym_db.RegisterMessage(Claim)
-18
View File
@@ -1,18 +0,0 @@
syntax = "proto2";
message Fee {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
enum Currency {
UNKNOWN_CURRENCY = 0;
LBC = 1;
BTC = 2;
USD = 3;
}
required Version version = 1;
required Currency currency = 2;
required bytes address = 3;
required float amount = 4;
}
+19 -19
View File
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/fee.proto
# source: fee.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/fee.proto',
package='',
name='fee.proto',
package='pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n\x1elbrynet/schema/proto/fee.proto\"\xcf\x01\n\x03\x46\x65\x65\x12\x1d\n\x07version\x18\x01 \x02(\x0e\x32\x0c.Fee.Version\x12\x1f\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\r.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
serialized_pb=_b('\n\tfee.proto\x12\x02pb\"\xd5\x01\n\x03\x46\x65\x65\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.pb.Fee.Version\x12\"\n\x08\x63urrency\x18\x02 \x02(\x0e\x32\x10.pb.Fee.Currency\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x04 \x02(\x02\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\";\n\x08\x43urrency\x12\x14\n\x10UNKNOWN_CURRENCY\x10\x00\x12\x07\n\x03LBC\x10\x01\x12\x07\n\x03\x42TC\x10\x02\x12\x07\n\x03USD\x10\x03')
)
_FEE_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Fee.Version',
full_name='pb.Fee.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -41,14 +41,14 @@ _FEE_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=139,
serialized_end=181,
serialized_start=128,
serialized_end=170,
)
_sym_db.RegisterEnumDescriptor(_FEE_VERSION)
_FEE_CURRENCY = _descriptor.EnumDescriptor(
name='Currency',
full_name='Fee.Currency',
full_name='pb.Fee.Currency',
filename=None,
file=DESCRIPTOR,
values=[
@@ -71,42 +71,42 @@ _FEE_CURRENCY = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=183,
serialized_end=242,
serialized_start=172,
serialized_end=231,
)
_sym_db.RegisterEnumDescriptor(_FEE_CURRENCY)
_FEE = _descriptor.Descriptor(
name='Fee',
full_name='Fee',
full_name='pb.Fee',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Fee.version', index=0,
name='version', full_name='pb.Fee.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='currency', full_name='Fee.currency', index=1,
name='currency', full_name='pb.Fee.currency', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='address', full_name='Fee.address', index=2,
name='address', full_name='pb.Fee.address', index=2,
number=3, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='amount', full_name='Fee.amount', index=3,
name='amount', full_name='pb.Fee.amount', index=3,
number=4, type=2, cpp_type=6, label=2,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
@@ -126,8 +126,8 @@ _FEE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=35,
serialized_end=242,
serialized_start=18,
serialized_end=231,
)
_FEE.fields_by_name['version'].enum_type = _FEE_VERSION
@@ -139,8 +139,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
Fee = _reflection.GeneratedProtocolMessageType('Fee', (_message.Message,), dict(
DESCRIPTOR = _FEE,
__module__ = 'lbrynet.schema.proto.fee_pb2'
# @@protoc_insertion_point(class_scope:Fee)
__module__ = 'fee_pb2'
# @@protoc_insertion_point(class_scope:pb.Fee)
))
_sym_db.RegisterMessage(Fee)
-212
View File
@@ -1,212 +0,0 @@
syntax = "proto2";
import "lbrynet/schema/proto/fee.proto";
message Metadata {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
_0_0_2 = 2;
_0_0_3 = 3;
_0_1_0 = 4;
}
enum Language {
UNKNOWN_LANGUAGE = 0;
en = 1;
aa = 2;
ab = 3;
ae = 4;
af = 5;
ak = 6;
am = 7;
an = 8;
ar = 9;
as = 10;
av = 11;
ay = 12;
az = 13;
ba = 14;
be = 15;
bg = 16;
bh = 17;
bi = 18;
bm = 19;
bn = 20;
bo = 21;
br = 22;
bs = 23;
ca = 24;
ce = 25;
ch = 26;
co = 27;
cr = 28;
cs = 29;
cu = 30;
cv = 31;
cy = 32;
da = 33;
de = 34;
dv = 35;
dz = 36;
ee = 37;
el = 38;
eo = 39;
es = 40;
et = 41;
eu = 42;
fa = 43;
ff = 44;
fi = 45;
fj = 46;
fo = 47;
fr = 48;
fy = 49;
ga = 50;
gd = 51;
gl = 52;
gn = 53;
gu = 54;
gv = 55;
ha = 56;
he = 57;
hi = 58;
ho = 59;
hr = 60;
ht = 61;
hu = 62;
hy = 63;
hz = 64;
ia = 65;
id = 66;
ie = 67;
ig = 68;
ii = 69;
ik = 70;
io = 71;
is = 72;
it = 73;
iu = 74;
ja = 75;
jv = 76;
ka = 77;
kg = 78;
ki = 79;
kj = 80;
kk = 81;
kl = 82;
km = 83;
kn = 84;
ko = 85;
kr = 86;
ks = 87;
ku = 88;
kv = 89;
kw = 90;
ky = 91;
la = 92;
lb = 93;
lg = 94;
li = 95;
ln = 96;
lo = 97;
lt = 98;
lu = 99;
lv = 100;
mg = 101;
mh = 102;
mi = 103;
mk = 104;
ml = 105;
mn = 106;
mr = 107;
ms = 108;
mt = 109;
my = 110;
na = 111;
nb = 112;
nd = 113;
ne = 114;
ng = 115;
nl = 116;
nn = 117;
no = 118;
nr = 119;
nv = 120;
ny = 121;
oc = 122;
oj = 123;
om = 124;
or = 125;
os = 126;
pa = 127;
pi = 128;
pl = 129;
ps = 130;
pt = 131;
qu = 132;
rm = 133;
rn = 134;
ro = 135;
ru = 136;
rw = 137;
sa = 138;
sc = 139;
sd = 140;
se = 141;
sg = 142;
si = 143;
sk = 144;
sl = 145;
sm = 146;
sn = 147;
so = 148;
sq = 149;
sr = 150;
ss = 151;
st = 152;
su = 153;
sv = 154;
sw = 155;
ta = 156;
te = 157;
tg = 158;
th = 159;
ti = 160;
tk = 161;
tl = 162;
tn = 163;
to = 164;
tr = 165;
ts = 166;
tt = 167;
tw = 168;
ty = 169;
ug = 170;
uk = 171;
ur = 172;
uz = 173;
ve = 174;
vi = 175;
vo = 176;
wa = 177;
wo = 178;
xh = 179;
yi = 180;
yo = 181;
za = 182;
zh = 183;
zu = 184;
}
required Version version = 1;
required Language language = 2;
required string title = 3;
required string description = 4;
required string author = 5;
required string license = 6;
required bool nsfw = 7;
optional Fee fee = 8;
optional string thumbnail = 9;
optional string preview = 10;
optional string licenseUrl = 11;
}
+36 -29
View File
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/metadata.proto
# source: metadata.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
from lbrynet.schema.proto import fee_pb2 as lbrynet_dot_schema_dot_proto_dot_fee__pb2
from . import fee_pb2 as fee__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/metadata.proto',
package='',
name='metadata.proto',
package='pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n#lbrynet/schema/proto/metadata.proto\x1a\x1elbrynet/schema/proto/fee.proto\"\xde\x0e\n\x08Metadata\x12\"\n\x07version\x18\x01 \x02(\x0e\x32\x11.Metadata.Version\x12$\n\x08language\x18\x02 \x02(\x0e\x32\x12.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x11\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x04.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\x01\x12\x07\n\x02ty\x1Line truncated
serialized_pb=_b('\n\x0emetadata.proto\x12\x02pb\x1a\tfee.proto\"\xfc\x0e\n\x08Metadata\x12%\n\x07version\x18\x01 \x02(\x0e\x32\x14.pb.Metadata.Version\x12\'\n\x08language\x18\x02 \x02(\x0e\x32\x15.pb.Metadata.Language\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x02(\t\x12\x0e\n\x06\x61uthor\x18\x05 \x02(\t\x12\x0f\n\x07license\x18\x06 \x02(\t\x12\x0c\n\x04nsfw\x18\x07 \x02(\x08\x12\x14\n\x03\x66\x65\x65\x18\x08 \x01(\x0b\x32\x07.pb.Fee\x12\x11\n\tthumbnail\x18\t \x01(\t\x12\x0f\n\x07preview\x18\n \x01(\t\x12\x12\n\nlicenseUrl\x18\x0b \x01(\t\x12\x13\n\x0breleaseTime\x18\x0c \x01(\x12\"N\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\x12\n\n\x06_0_0_2\x10\x02\x12\n\n\x06_0_0_3\x10\x03\x12\n\n\x06_0_1_0\x10\x04\"\x99\x0c\n\x08Language\x12\x14\n\x10UNKNOWN_LANGUAGE\x10\x00\x12\x06\n\x02\x65n\x10\x01\x12\x06\n\x02\x61\x61\x10\x02\x12\x06\n\x02\x61\x62\x10\x03\x12\x06\n\x02\x61\x65\x10\x04\x12\x06\n\x02\x61\x66\x10\x05\x12\x06\n\x02\x61k\x10\x06\x12\x06\n\x02\x61m\x10\x07\x12\x06\n\x02\x61n\x10\x08\x12\x06\n\x02\x61r\x10\t\x12\x06\n\x02\x61s\x10\n\x12\x06\n\x02\x61v\x10\x0b\x12\x06\n\x02\x61y\x10\x0c\x12\x06\n\x02\x61z\x10\r\x12\x06\n\x02\x62\x61\x10\x0e\x12\x06\n\x02\x62\x65\x10\x0f\x12\x06\n\x02\x62g\x10\x10\x12\x06\n\x02\x62h\x10\x11\x12\x06\n\x02\x62i\x10\x12\x12\x06\n\x02\x62m\x10\x13\x12\x06\n\x02\x62n\x10\x14\x12\x06\n\x02\x62o\x10\x15\x12\x06\n\x02\x62r\x10\x16\x12\x06\n\x02\x62s\x10\x17\x12\x06\n\x02\x63\x61\x10\x18\x12\x06\n\x02\x63\x65\x10\x19\x12\x06\n\x02\x63h\x10\x1a\x12\x06\n\x02\x63o\x10\x1b\x12\x06\n\x02\x63r\x10\x1c\x12\x06\n\x02\x63s\x10\x1d\x12\x06\n\x02\x63u\x10\x1e\x12\x06\n\x02\x63v\x10\x1f\x12\x06\n\x02\x63y\x10 \x12\x06\n\x02\x64\x61\x10!\x12\x06\n\x02\x64\x65\x10\"\x12\x06\n\x02\x64v\x10#\x12\x06\n\x02\x64z\x10$\x12\x06\n\x02\x65\x65\x10%\x12\x06\n\x02\x65l\x10&\x12\x06\n\x02\x65o\x10\'\x12\x06\n\x02\x65s\x10(\x12\x06\n\x02\x65t\x10)\x12\x06\n\x02\x65u\x10*\x12\x06\n\x02\x66\x61\x10+\x12\x06\n\x02\x66\x66\x10,\x12\x06\n\x02\x66i\x10-\x12\x06\n\x02\x66j\x10.\x12\x06\n\x02\x66o\x10/\x12\x06\n\x02\x66r\x10\x30\x12\x06\n\x02\x66y\x10\x31\x12\x06\n\x02ga\x10\x32\x12\x06\n\x02gd\x10\x33\x12\x06\n\x02gl\x10\x34\x12\x06\n\x02gn\x10\x35\x12\x06\n\x02gu\x10\x36\x12\x06\n\x02gv\x10\x37\x12\x06\n\x02ha\x10\x38\x12\x06\n\x02he\x10\x39\x12\x06\n\x02hi\x10:\x12\x06\n\x02ho\x10;\x12\x06\n\x02hr\x10<\x12\x06\n\x02ht\x10=\x12\x06\n\x02hu\x10>\x12\x06\n\x02hy\x10?\x12\x06\n\x02hz\x10@\x12\x06\n\x02ia\x10\x41\x12\x06\n\x02id\x10\x42\x12\x06\n\x02ie\x10\x43\x12\x06\n\x02ig\x10\x44\x12\x06\n\x02ii\x10\x45\x12\x06\n\x02ik\x10\x46\x12\x06\n\x02io\x10G\x12\x06\n\x02is\x10H\x12\x06\n\x02it\x10I\x12\x06\n\x02iu\x10J\x12\x06\n\x02ja\x10K\x12\x06\n\x02jv\x10L\x12\x06\n\x02ka\x10M\x12\x06\n\x02kg\x10N\x12\x06\n\x02ki\x10O\x12\x06\n\x02kj\x10P\x12\x06\n\x02kk\x10Q\x12\x06\n\x02kl\x10R\x12\x06\n\x02km\x10S\x12\x06\n\x02kn\x10T\x12\x06\n\x02ko\x10U\x12\x06\n\x02kr\x10V\x12\x06\n\x02ks\x10W\x12\x06\n\x02ku\x10X\x12\x06\n\x02kv\x10Y\x12\x06\n\x02kw\x10Z\x12\x06\n\x02ky\x10[\x12\x06\n\x02la\x10\\\x12\x06\n\x02lb\x10]\x12\x06\n\x02lg\x10^\x12\x06\n\x02li\x10_\x12\x06\n\x02ln\x10`\x12\x06\n\x02lo\x10\x61\x12\x06\n\x02lt\x10\x62\x12\x06\n\x02lu\x10\x63\x12\x06\n\x02lv\x10\x64\x12\x06\n\x02mg\x10\x65\x12\x06\n\x02mh\x10\x66\x12\x06\n\x02mi\x10g\x12\x06\n\x02mk\x10h\x12\x06\n\x02ml\x10i\x12\x06\n\x02mn\x10j\x12\x06\n\x02mr\x10k\x12\x06\n\x02ms\x10l\x12\x06\n\x02mt\x10m\x12\x06\n\x02my\x10n\x12\x06\n\x02na\x10o\x12\x06\n\x02nb\x10p\x12\x06\n\x02nd\x10q\x12\x06\n\x02ne\x10r\x12\x06\n\x02ng\x10s\x12\x06\n\x02nl\x10t\x12\x06\n\x02nn\x10u\x12\x06\n\x02no\x10v\x12\x06\n\x02nr\x10w\x12\x06\n\x02nv\x10x\x12\x06\n\x02ny\x10y\x12\x06\n\x02oc\x10z\x12\x06\n\x02oj\x10{\x12\x06\n\x02om\x10|\x12\x06\n\x02or\x10}\x12\x06\n\x02os\x10~\x12\x06\n\x02pa\x10\x7f\x12\x07\n\x02pi\x10\x80\x01\x12\x07\n\x02pl\x10\x81\x01\x12\x07\n\x02ps\x10\x82\x01\x12\x07\n\x02pt\x10\x83\x01\x12\x07\n\x02qu\x10\x84\x01\x12\x07\n\x02rm\x10\x85\x01\x12\x07\n\x02rn\x10\x86\x01\x12\x07\n\x02ro\x10\x87\x01\x12\x07\n\x02ru\x10\x88\x01\x12\x07\n\x02rw\x10\x89\x01\x12\x07\n\x02sa\x10\x8a\x01\x12\x07\n\x02sc\x10\x8b\x01\x12\x07\n\x02sd\x10\x8c\x01\x12\x07\n\x02se\x10\x8d\x01\x12\x07\n\x02sg\x10\x8e\x01\x12\x07\n\x02si\x10\x8f\x01\x12\x07\n\x02sk\x10\x90\x01\x12\x07\n\x02sl\x10\x91\x01\x12\x07\n\x02sm\x10\x92\x01\x12\x07\n\x02sn\x10\x93\x01\x12\x07\n\x02so\x10\x94\x01\x12\x07\n\x02sq\x10\x95\x01\x12\x07\n\x02sr\x10\x96\x01\x12\x07\n\x02ss\x10\x97\x01\x12\x07\n\x02st\x10\x98\x01\x12\x07\n\x02su\x10\x99\x01\x12\x07\n\x02sv\x10\x9a\x01\x12\x07\n\x02sw\x10\x9b\x01\x12\x07\n\x02ta\x10\x9c\x01\x12\x07\n\x02te\x10\x9d\x01\x12\x07\n\x02tg\x10\x9e\x01\x12\x07\n\x02th\x10\x9f\x01\x12\x07\n\x02ti\x10\xa0\x01\x12\x07\n\x02tk\x10\xa1\x01\x12\x07\n\x02tl\x10\xa2\x01\x12\x07\n\x02tn\x10\xa3\x01\x12\x07\n\x02to\x10\xa4\x01\x12\x07\n\x02tr\x10\xa5\x01\x12\x07\n\x02ts\x10\xa6\x01\x12\x07\n\x02tt\x10\xa7\x01\x12\x07\n\x02tw\x10\xa8\xLine truncated
,
dependencies=[lbrynet_dot_schema_dot_proto_dot_fee__pb2.DESCRIPTOR,])
dependencies=[fee__pb2.DESCRIPTOR,])
_METADATA_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Metadata.Version',
full_name='pb.Metadata.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -55,14 +55,14 @@ _METADATA_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=316,
serialized_end=394,
serialized_start=308,
serialized_end=386,
)
_sym_db.RegisterEnumDescriptor(_METADATA_VERSION)
_METADATA_LANGUAGE = _descriptor.EnumDescriptor(
name='Language',
full_name='Metadata.Language',
full_name='pb.Metadata.Language',
filename=None,
file=DESCRIPTOR,
values=[
@@ -809,96 +809,103 @@ _METADATA_LANGUAGE = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=397,
serialized_end=1958,
serialized_start=389,
serialized_end=1950,
)
_sym_db.RegisterEnumDescriptor(_METADATA_LANGUAGE)
_METADATA = _descriptor.Descriptor(
name='Metadata',
full_name='Metadata',
full_name='pb.Metadata',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Metadata.version', index=0,
name='version', full_name='pb.Metadata.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='language', full_name='Metadata.language', index=1,
name='language', full_name='pb.Metadata.language', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='title', full_name='Metadata.title', index=2,
name='title', full_name='pb.Metadata.title', index=2,
number=3, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='description', full_name='Metadata.description', index=3,
name='description', full_name='pb.Metadata.description', index=3,
number=4, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='author', full_name='Metadata.author', index=4,
name='author', full_name='pb.Metadata.author', index=4,
number=5, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='license', full_name='Metadata.license', index=5,
name='license', full_name='pb.Metadata.license', index=5,
number=6, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='nsfw', full_name='Metadata.nsfw', index=6,
name='nsfw', full_name='pb.Metadata.nsfw', index=6,
number=7, type=8, cpp_type=7, label=2,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='fee', full_name='Metadata.fee', index=7,
name='fee', full_name='pb.Metadata.fee', index=7,
number=8, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='thumbnail', full_name='Metadata.thumbnail', index=8,
name='thumbnail', full_name='pb.Metadata.thumbnail', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='preview', full_name='Metadata.preview', index=9,
name='preview', full_name='pb.Metadata.preview', index=9,
number=10, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='licenseUrl', full_name='Metadata.licenseUrl', index=10,
name='licenseUrl', full_name='pb.Metadata.licenseUrl', index=10,
number=11, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='releaseTime', full_name='pb.Metadata.releaseTime', index=11,
number=12, type=18, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
@@ -913,13 +920,13 @@ _METADATA = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=72,
serialized_end=1958,
serialized_start=34,
serialized_end=1950,
)
_METADATA.fields_by_name['version'].enum_type = _METADATA_VERSION
_METADATA.fields_by_name['language'].enum_type = _METADATA_LANGUAGE
_METADATA.fields_by_name['fee'].message_type = lbrynet_dot_schema_dot_proto_dot_fee__pb2._FEE
_METADATA.fields_by_name['fee'].message_type = fee__pb2._FEE
_METADATA_VERSION.containing_type = _METADATA
_METADATA_LANGUAGE.containing_type = _METADATA
DESCRIPTOR.message_types_by_name['Metadata'] = _METADATA
@@ -927,8 +934,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
Metadata = _reflection.GeneratedProtocolMessageType('Metadata', (_message.Message,), dict(
DESCRIPTOR = _METADATA,
__module__ = 'lbrynet.schema.proto.metadata_pb2'
# @@protoc_insertion_point(class_scope:Metadata)
__module__ = 'metadata_pb2'
# @@protoc_insertion_point(class_scope:pb.Metadata)
))
_sym_db.RegisterMessage(Metadata)
-14
View File
@@ -1,14 +0,0 @@
syntax = "proto2";
import "lbrynet/schema/proto/certificate.proto";
message Signature {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
required Version version = 1;
required KeyType signatureType = 2;
required bytes signature = 3;
required bytes certificateId = 4;
}
+19 -19
View File
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/signature.proto
# source: signature.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -12,23 +12,23 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
from lbrynet.schema.proto import certificate_pb2 as lbrynet_dot_schema_dot_proto_dot_certificate__pb2
from . import certificate_pb2 as certificate__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/signature.proto',
package='',
name='signature.proto',
package='pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n$lbrynet/schema/proto/signature.proto\x1a&lbrynet/schema/proto/certificate.proto\"\xa7\x01\n\tSignature\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.Signature.Version\x12\x1f\n\rsignatureType\x18\x02 \x02(\x0e\x32\x08.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
serialized_pb=_b('\n\x0fsignature.proto\x12\x02pb\x1a\x11\x63\x65rtificate.proto\"\xad\x01\n\tSignature\x12&\n\x07version\x18\x01 \x02(\x0e\x32\x15.pb.Signature.Version\x12\"\n\rsignatureType\x18\x02 \x02(\x0e\x32\x0b.pb.KeyType\x12\x11\n\tsignature\x18\x03 \x02(\x0c\x12\x15\n\rcertificateId\x18\x04 \x02(\x0c\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
,
dependencies=[lbrynet_dot_schema_dot_proto_dot_certificate__pb2.DESCRIPTOR,])
dependencies=[certificate__pb2.DESCRIPTOR,])
_SIGNATURE_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Signature.Version',
full_name='pb.Signature.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -43,42 +43,42 @@ _SIGNATURE_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=206,
serialized_end=248,
serialized_start=174,
serialized_end=216,
)
_sym_db.RegisterEnumDescriptor(_SIGNATURE_VERSION)
_SIGNATURE = _descriptor.Descriptor(
name='Signature',
full_name='Signature',
full_name='pb.Signature',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Signature.version', index=0,
name='version', full_name='pb.Signature.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='signatureType', full_name='Signature.signatureType', index=1,
name='signatureType', full_name='pb.Signature.signatureType', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='signature', full_name='Signature.signature', index=2,
name='signature', full_name='pb.Signature.signature', index=2,
number=3, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='certificateId', full_name='Signature.certificateId', index=3,
name='certificateId', full_name='pb.Signature.certificateId', index=3,
number=4, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
@@ -97,20 +97,20 @@ _SIGNATURE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=81,
serialized_end=248,
serialized_start=43,
serialized_end=216,
)
_SIGNATURE.fields_by_name['version'].enum_type = _SIGNATURE_VERSION
_SIGNATURE.fields_by_name['signatureType'].enum_type = lbrynet_dot_schema_dot_proto_dot_certificate__pb2._KEYTYPE
_SIGNATURE.fields_by_name['signatureType'].enum_type = certificate__pb2._KEYTYPE
_SIGNATURE_VERSION.containing_type = _SIGNATURE
DESCRIPTOR.message_types_by_name['Signature'] = _SIGNATURE
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Signature = _reflection.GeneratedProtocolMessageType('Signature', (_message.Message,), dict(
DESCRIPTOR = _SIGNATURE,
__module__ = 'lbrynet.schema.proto.signature_pb2'
# @@protoc_insertion_point(class_scope:Signature)
__module__ = 'signature_pb2'
# @@protoc_insertion_point(class_scope:pb.Signature)
))
_sym_db.RegisterMessage(Signature)
-16
View File
@@ -1,16 +0,0 @@
syntax = "proto2";
message Source {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
required Version version = 1;
enum SourceTypes {
UNKNOWN_SOURCE_TYPE = 0;
lbry_sd_hash = 1;
}
required SourceTypes sourceType = 2;
required bytes source = 3;
required string contentType = 4;
}
+19 -19
View File
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/source.proto
# source: source.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -15,18 +15,18 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/source.proto',
package='',
name='source.proto',
package='pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n!lbrynet/schema/proto/source.proto\"\xde\x01\n\x06Source\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Source.Version\x12\'\n\nsourceType\x18\x02 \x02(\x0e\x32\x13.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
serialized_pb=_b('\n\x0csource.proto\x12\x02pb\"\xe4\x01\n\x06Source\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.pb.Source.Version\x12*\n\nsourceType\x18\x02 \x02(\x0e\x32\x16.pb.Source.SourceTypes\x12\x0e\n\x06source\x18\x03 \x02(\x0c\x12\x13\n\x0b\x63ontentType\x18\x04 \x02(\t\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01\"8\n\x0bSourceTypes\x12\x17\n\x13UNKNOWN_SOURCE_TYPE\x10\x00\x12\x10\n\x0clbry_sd_hash\x10\x01')
)
_SOURCE_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Source.Version',
full_name='pb.Source.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -41,14 +41,14 @@ _SOURCE_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=160,
serialized_end=202,
serialized_start=149,
serialized_end=191,
)
_sym_db.RegisterEnumDescriptor(_SOURCE_VERSION)
_SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
name='SourceTypes',
full_name='Source.SourceTypes',
full_name='pb.Source.SourceTypes',
filename=None,
file=DESCRIPTOR,
values=[
@@ -63,42 +63,42 @@ _SOURCE_SOURCETYPES = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=204,
serialized_end=260,
serialized_start=193,
serialized_end=249,
)
_sym_db.RegisterEnumDescriptor(_SOURCE_SOURCETYPES)
_SOURCE = _descriptor.Descriptor(
name='Source',
full_name='Source',
full_name='pb.Source',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Source.version', index=0,
name='version', full_name='pb.Source.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='sourceType', full_name='Source.sourceType', index=1,
name='sourceType', full_name='pb.Source.sourceType', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='source', full_name='Source.source', index=2,
name='source', full_name='pb.Source.source', index=2,
number=3, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='contentType', full_name='Source.contentType', index=3,
name='contentType', full_name='pb.Source.contentType', index=3,
number=4, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
@@ -118,8 +118,8 @@ _SOURCE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=38,
serialized_end=260,
serialized_start=21,
serialized_end=249,
)
_SOURCE.fields_by_name['version'].enum_type = _SOURCE_VERSION
@@ -131,8 +131,8 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
Source = _reflection.GeneratedProtocolMessageType('Source', (_message.Message,), dict(
DESCRIPTOR = _SOURCE,
__module__ = 'lbrynet.schema.proto.source_pb2'
# @@protoc_insertion_point(class_scope:Source)
__module__ = 'source_pb2'
# @@protoc_insertion_point(class_scope:pb.Source)
))
_sym_db.RegisterMessage(Source)
-15
View File
@@ -1,15 +0,0 @@
syntax = "proto2";
import "lbrynet/schema/proto/metadata.proto";
import "lbrynet/schema/proto/source.proto";
message Stream {
enum Version {
UNKNOWN_VERSION = 0;
_0_0_1 = 1;
}
required Version version = 1;
required Metadata metadata = 2;
required Source source = 3;
}
+20 -20
View File
@@ -1,5 +1,5 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: lbrynet/schema/proto/stream.proto
# source: stream.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
@@ -12,24 +12,24 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
from lbrynet.schema.proto import metadata_pb2 as lbrynet_dot_schema_dot_proto_dot_metadata__pb2
from lbrynet.schema.proto import source_pb2 as lbrynet_dot_schema_dot_proto_dot_source__pb2
from . import metadata_pb2 as metadata__pb2
from . import source_pb2 as source__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name='lbrynet/schema/proto/stream.proto',
package='',
name='stream.proto',
package='pb',
syntax='proto2',
serialized_options=None,
serialized_pb=_b('\n!lbrynet/schema/proto/stream.proto\x1a#lbrynet/schema/proto/metadata.proto\x1a!lbrynet/schema/proto/source.proto\"\x8c\x01\n\x06Stream\x12 \n\x07version\x18\x01 \x02(\x0e\x32\x0f.Stream.Version\x12\x1b\n\x08metadata\x18\x02 \x02(\x0b\x32\t.Metadata\x12\x17\n\x06source\x18\x03 \x02(\x0b\x32\x07.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
serialized_pb=_b('\n\x0cstream.proto\x12\x02pb\x1a\x0emetadata.proto\x1a\x0csource.proto\"\x95\x01\n\x06Stream\x12#\n\x07version\x18\x01 \x02(\x0e\x32\x12.pb.Stream.Version\x12\x1e\n\x08metadata\x18\x02 \x02(\x0b\x32\x0c.pb.Metadata\x12\x1a\n\x06source\x18\x03 \x02(\x0b\x32\n.pb.Source\"*\n\x07Version\x12\x13\n\x0fUNKNOWN_VERSION\x10\x00\x12\n\n\x06_0_0_1\x10\x01')
,
dependencies=[lbrynet_dot_schema_dot_proto_dot_metadata__pb2.DESCRIPTOR,lbrynet_dot_schema_dot_proto_dot_source__pb2.DESCRIPTOR,])
dependencies=[metadata__pb2.DESCRIPTOR,source__pb2.DESCRIPTOR,])
_STREAM_VERSION = _descriptor.EnumDescriptor(
name='Version',
full_name='Stream.Version',
full_name='pb.Stream.Version',
filename=None,
file=DESCRIPTOR,
values=[
@@ -44,35 +44,35 @@ _STREAM_VERSION = _descriptor.EnumDescriptor(
],
containing_type=None,
serialized_options=None,
serialized_start=208,
serialized_end=250,
serialized_start=158,
serialized_end=200,
)
_sym_db.RegisterEnumDescriptor(_STREAM_VERSION)
_STREAM = _descriptor.Descriptor(
name='Stream',
full_name='Stream',
full_name='pb.Stream',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='version', full_name='Stream.version', index=0,
name='version', full_name='pb.Stream.version', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='metadata', full_name='Stream.metadata', index=1,
name='metadata', full_name='pb.Stream.metadata', index=1,
number=2, type=11, cpp_type=10, label=2,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='source', full_name='Stream.source', index=2,
name='source', full_name='pb.Stream.source', index=2,
number=3, type=11, cpp_type=10, label=2,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
@@ -91,21 +91,21 @@ _STREAM = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=110,
serialized_end=250,
serialized_start=51,
serialized_end=200,
)
_STREAM.fields_by_name['version'].enum_type = _STREAM_VERSION
_STREAM.fields_by_name['metadata'].message_type = lbrynet_dot_schema_dot_proto_dot_metadata__pb2._METADATA
_STREAM.fields_by_name['source'].message_type = lbrynet_dot_schema_dot_proto_dot_source__pb2._SOURCE
_STREAM.fields_by_name['metadata'].message_type = metadata__pb2._METADATA
_STREAM.fields_by_name['source'].message_type = source__pb2._SOURCE
_STREAM_VERSION.containing_type = _STREAM
DESCRIPTOR.message_types_by_name['Stream'] = _STREAM
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict(
DESCRIPTOR = _STREAM,
__module__ = 'lbrynet.schema.proto.stream_pb2'
# @@protoc_insertion_point(class_scope:Stream)
__module__ = 'stream_pb2'
# @@protoc_insertion_point(class_scope:pb.Stream)
))
_sym_db.RegisterMessage(Stream)
+5 -1
View File
@@ -14,4 +14,8 @@ class Metadata(Schema):
if 'fee' in _metadata:
fee_pb = Fee.load(_metadata.pop('fee'))
_message_pb.fee.CopyFrom(fee_pb)
return cls._load(_metadata, _message_pb)
_message_pb.releaseTime = int(_metadata.get('releaseTime', 0))
built_message = cls._load(_metadata, _message_pb)
if built_message.releaseTime == 0:
built_message.ClearField('releaseTime')
return built_message
+3 -3
View File
@@ -50,11 +50,11 @@ class NIST_ECDSASigner(object):
digest = self.HASHFUNC(bytearray(b''.join(fields))).digest()
return self.private_key.sign_digest_deterministic(digest, hashfunc=self.HASHFUNC)
def sign_stream_claim(self, claim, claim_address, cert_claim_id, name, detached=False):
def sign_stream_claim(self, claim, claim_address, cert_claim_id, name, legacy=False):
validate_claim_id(cert_claim_id)
raw_cert_id = binascii.unhexlify(cert_claim_id)
decoded_addr = decode_address(claim_address)
if detached:
if not legacy:
assert name, "Name is required for detached signatures"
assert self.CURVE_NAME == SECP256k1, f"Only SECP256k1 is supported, not: {self.CURVE_NAME}"
signature = self.sign(
@@ -66,7 +66,7 @@ class NIST_ECDSASigner(object):
else:
signature = self.sign(decoded_addr, claim.serialized_no_signature, raw_cert_id)
if detached:
if not legacy:
return Claim.load(decode_b64_fields(claim.protobuf_dict)), Signature(NAMED_SECP256K1(
signature,
raw_cert_id,
-57
View File
@@ -1,57 +0,0 @@
"""Decrypt a single blob"""
import argparse
import binascii
import logging
import os
import sys
from twisted.internet import defer
from twisted.internet import reactor
from lbrynet import conf
from lbrynet.cryptstream import CryptBlob
from lbrynet.blob import BlobFile
from lbrynet.core import log_support
log = logging.getLogger('decrypt_blob')
def main():
conf.initialize_settings()
parser = argparse.ArgumentParser()
parser.add_argument('blob_file')
parser.add_argument('hex_key')
parser.add_argument('hex_iv')
parser.add_argument('output')
args = parser.parse_args()
log_support.configure_console()
d = run(args)
reactor.run()
@defer.inlineCallbacks
def run(args):
try:
yield decrypt_blob(args.blob_file, args.hex_key, args.hex_iv, args.output)
except Exception:
log.exception('Failed to decrypt blob')
finally:
reactor.callLater(0, reactor.stop)
@defer.inlineCallbacks
def decrypt_blob(blob_file, key, iv, output):
filename = os.path.abspath(blob_file)
length = os.path.getsize(filename)
directory, blob_hash = os.path.split(filename)
blob = BlobFile(directory, blob_hash, length)
decryptor = CryptBlob.StreamBlobDecryptor(
blob, binascii.unhexlify(key), binascii.unhexlify(iv), length)
with open(output, 'w') as f:
yield decryptor.decrypt(f.write)
if __name__ == '__main__':
sys.exit(main())
-85
View File
@@ -1,85 +0,0 @@
import curses
import time
import datetime
from jsonrpc.proxy import JSONRPCProxy
stdscr = curses.initscr()
api = JSONRPCProxy.from_url("http://localhost:5280")
def init_curses():
curses.noecho()
curses.cbreak()
stdscr.nodelay(1)
stdscr.keypad(1)
def teardown_curses():
curses.nocbreak()
stdscr.keypad(0)
curses.echo()
curses.endwin()
def refresh(node_index):
height, width = stdscr.getmaxyx()
node_ids = api.get_node_ids()
node_id = node_ids[node_index]
node_statuses = api.node_status()
running = node_statuses[node_id]
buckets = api.node_routing_table(node_id=node_id)
for y in range(height):
stdscr.addstr(y, 0, " " * (width - 1))
stdscr.addstr(0, 0, "node id: %s, running: %s (%i/%i running)" % (node_id, running, sum(node_statuses.values()), len(node_ids)))
stdscr.addstr(1, 0, "%i buckets, %i contacts" %
(len(buckets), sum([len(buckets[b]['contacts']) for b in buckets])))
y = 3
for i in sorted(buckets.keys()):
stdscr.addstr(y, 0, "bucket %s" % i)
y += 1
for h in sorted(buckets[i]['contacts'], key=lambda x: x['node_id'].decode('hex')):
stdscr.addstr(y, 0, '%s (%s:%i) failures: %i, last replied to us: %s, last requested from us: %s' %
(h['node_id'], h['address'], h['port'], h['failedRPCs'],
datetime.datetime.fromtimestamp(float(h['lastReplied'] or 0)),
datetime.datetime.fromtimestamp(float(h['lastRequested'] or 0))))
y += 1
y += 1
stdscr.addstr(y + 1, 0, str(time.time()))
stdscr.refresh()
return len(node_ids)
def do_main():
c = None
nodes = 1
node_index = 0
while c not in [ord('q'), ord('Q')]:
try:
nodes = refresh(node_index)
except:
pass
c = stdscr.getch()
if c == curses.KEY_LEFT:
node_index -= 1
node_index = max(node_index, 0)
elif c == curses.KEY_RIGHT:
node_index += 1
node_index = min(node_index, nodes - 1)
time.sleep(0.1)
def main():
try:
init_curses()
do_main()
finally:
teardown_curses()
if __name__ == "__main__":
main()
-104
View File
@@ -1,104 +0,0 @@
"""A simple script that attempts to directly download a single blob or stream from a given peer"""
import argparse
import logging
import sys
import tempfile
import time
import shutil
from pprint import pprint
from twisted.internet import asyncioreactor
asyncioreactor.install()
from twisted.internet import defer, threads, reactor
from lbrynet import conf, log_support
from lbrynet.p2p import Peer
from lbrynet.p2p.SinglePeerDownloader import SinglePeerDownloader
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorReader
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.extras.daemon.Components import f2d
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.wallet import LbryWalletManager
log = logging.getLogger()
def main(args=None):
conf.initialize_settings()
parser = argparse.ArgumentParser()
parser.add_argument('peer')
parser.add_argument('blob_hash')
parser.add_argument('--timeout', type=int, default=30)
args = parser.parse_args(args)
log_support.configure_console(level='DEBUG')
log_support.configure_twisted()
if ":" in str(args.peer):
host, port = str(args.peer).strip().split(":")
else:
host = args.peer
port = 3333
d = download_it(Peer.Peer(host, int(port)), args.timeout, args.blob_hash)
d.addErrback(log.exception)
d.addBoth(lambda _: reactor.callLater(0, reactor.stop))
reactor.run()
@defer.inlineCallbacks
def download_it(peer, timeout, blob_hash):
tmp_dir = yield threads.deferToThread(tempfile.mkdtemp)
storage = SQLiteStorage(tmp_dir, reactor)
yield storage.setup()
tmp_blob_manager = DiskBlobManager(tmp_dir, storage)
config = {'auto_connect': True}
config['wallet_dir'] = tempfile.mkdtemp()
config['use_keyring'] = False
config['blockchain_name'] = conf.settings['blockchain_name']
config['lbryum_servers'] = []
wallet = yield f2d(LbryWalletManager.from_lbrynet_config(config, storage))
downloader = SinglePeerDownloader()
downloader.setup(wallet)
try:
blob_downloaded = yield downloader.download_blob_from_peer(peer, timeout, blob_hash,
tmp_blob_manager)
if blob_downloaded:
log.info("SUCCESS!")
blob = yield tmp_blob_manager.get_blob(blob_hash)
pprint(blob)
if not blob.verified:
log.error("except that its not verified....")
else:
reader = BlobStreamDescriptorReader(blob)
info = None
for x in range(0, 3):
try:
info = yield reader.get_info()
except ValueError:
pass
if info:
break
# there's some kind of race condition where it sometimes doesn't write the blob to disk in time
time.sleep(0.1)
if info is not None:
pprint(info)
for content_blob in info['blobs']:
if 'blob_hash' in content_blob:
yield download_it(peer, timeout, content_blob['blob_hash'])
else:
log.error("Download failed")
finally:
yield tmp_blob_manager.stop()
yield threads.deferToThread(shutil.rmtree, tmp_dir)
defer.returnValue(True)
if __name__ == '__main__':
sys.exit(main())
-80
View File
@@ -1,80 +0,0 @@
"""A test script that downloads blobs from a reflector server"""
import argparse
import itertools
import json
import random
import subprocess
import sys
def main():
parser = argparse.ArgumentParser()
parser.add_argument('reflector_ip')
parser.add_argument('--ssh-key')
parser.add_argument('--size', type=int, default=100)
parser.add_argument('--batch', type=int, default=10)
parser.add_argument('--timeout', type=int, default=30)
parser.add_argument('--hashes', help='file listing hashes in json')
args = parser.parse_args()
if args.hashes:
hashes = readHashes(args.hashes)
else:
hashes = getHashes(args.reflector_ip, args.ssh_key)
if len(hashes) > args.size:
selected_hashes = random.sample(hashes, args.size)
else:
print 'Only {} hashes are available'.format(hashes)
selected_hashes = hashes
successes = 0
for hashes in grouper(selected_hashes, args.batch):
hashes = filter(None, hashes)
successes += downloadHashes(args.reflector_ip, hashes, args.timeout)
print 'Downloaded {} / {}'.format(successes, len(selected_hashes))
def grouper(iterable, n, fillvalue=None):
"Collect data into fixed-length chunks or blocks"
# grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
args = [iter(iterable)] * n
return itertools.izip_longest(fillvalue=fillvalue, *args)
def readHashes(hash_file):
with open(hash_file) as f:
return json.load(f)
def getHashes(ip, key=None):
key = ['-i', key] if key else []
hashes = subprocess.check_output(['ssh'] + key +
['lbry@{}'.format(ip), '/opt/venvs/lbrynet/bin/lbrynet-cli', 'get_blob_hashes'])
return json.loads(hashes)
def downloadHashes(ip, blob_hashes, timeout=30):
processes = [
subprocess.Popen(
[
'python',
'download_blob_from_peer.py',
'--timeout', str(timeout), '{}:3333'.format(ip), blob_hash,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
for blob_hash in blob_hashes
]
for p, h in zip(processes, blob_hashes):
stdout, stderr = p.communicate()
print p.returncode, h
if p.returncode != 0:
print 'Failed to download', h
print stdout
print stderr
return sum(1 for p in processes if p.returncode == 0)
if __name__ == '__main__':
sys.exit(main())
-63
View File
@@ -1,63 +0,0 @@
"""Encrypt a single file using the given key and iv"""
import argparse
import logging
import sys
from twisted.internet import defer
from twisted.internet import reactor
from twisted.web.client import FileBodyProducer
from lbrynet import conf
from lbrynet.core import log_support
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
from lbrynet.core.BlobManager import DiskBlobManager
from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
log = logging.getLogger('decrypt_blob')
def main():
conf.initialize_settings()
parser = argparse.ArgumentParser()
parser.add_argument('filename')
parser.add_argument('hex_key')
parser.add_argument('hex_iv')
args = parser.parse_args()
log_support.configure_console(level='DEBUG')
run(args)
reactor.run()
@defer.inlineCallbacks
def run(args):
try:
yield encrypt_blob(args.filename, args.hex_key, args.hex_iv)
except Exception:
log.exception('Failed to encrypt blob')
finally:
reactor.callLater(0, reactor.stop)
@defer.inlineCallbacks
def encrypt_blob(filename, key, iv):
dummy_announcer = DummyHashAnnouncer()
manager = DiskBlobManager(dummy_announcer, '.', '.')
yield manager.setup()
creator = CryptStreamCreator(manager, filename, key, iv_generator(iv))
with open(filename, 'r') as infile:
producer = FileBodyProducer(infile, readSize=2**22)
yield producer.startProducing(creator)
yield creator.stop()
def iv_generator(iv):
iv = int(iv, 16)
while 1:
iv += 1
yield ("%016d" % iv)[-16:]
if __name__ == '__main__':
sys.exit(main())
-206
View File
@@ -1,206 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Generate docs: python gen_api_docs.py
# See docs: pip install mkdocs; mkdocs serve
# Push docs: mkdocs build
import re
import inspect
import subprocess
import os
import sys
from lbrynet.daemon.Daemon import Daemon
import pip
installed_packages = [package.project_name for package in pip.get_installed_distributions()]
for package in ["mkdocs", "mkdocs-material"]:
if package not in installed_packages:
print "'" + package + "' is not installed"
sys.exit(1)
try:
from tabulate import tabulate
except ImportError:
raise ImportError("tabulate is not installed")
INDENT = " "
REQD_CMD_REGEX = r"\(.*?=<(?P<reqd>.*?)>\)"
OPT_CMD_REGEX = r"\[.*?=<(?P<opt>.*?)>\]"
CMD_REGEX = r"--.*?(?P<cmd>.*?)[=,\s,<]"
DOCS_BUILD_DIR = "docs_build" # must match mkdocs.yml
def _cli_tabulate_options(_options_docstr, method):
_option_list = []
for line in _options_docstr.splitlines():
if (line.strip().startswith("--")):
# separates command name and description
parts = line.split(":", 1)
# separates command type(in brackets) and description
new_parts = parts[1].lstrip().split(" ", 1)
else:
parts = [line]
# len will be 2 when there's cmd name and description
if len(parts) == 2:
_option_list.append([parts[0], ":", new_parts[0], new_parts[1]])
# len will be 1 when there's continuation of multiline description in the next line
# check `blob_announce`'s `stream_hash` command
elif len(parts) == 1:
_option_list.append([None, None, None, parts[0]])
else:
print "Error: Ill formatted doc string for {}".format(method)
print "Error causing line: {}".format(line)
# tabulate to make the options look pretty
_options_docstr_no_indent = tabulate(_option_list, missingval="", tablefmt="plain")
# Indent the options properly
_options_docstr = ""
for line in _options_docstr_no_indent.splitlines():
_options_docstr += INDENT + line + '\n'
return _options_docstr
def _api_tabulate_options(_options_docstr, method, reqd_matches, opt_matches):
_option_list = []
for line in _options_docstr.splitlines():
if (line.strip().startswith("--")):
# separates command name and description
parts = line.split(":", 1)
# checks whether the command is optional or required
# and remove the cli type formatting and convert to
# api style formatitng
match = re.findall(CMD_REGEX, parts[0])
if match[0] not in reqd_matches:
parts[0] = "'" + match[0] + "'"
else:
parts[0] = "'" + match[0] + "' (required)"
# separates command type(in brackets) and description
new_parts = parts[1].lstrip().split(" ", 1)
else:
parts = [line]
# len will be 2 when there's cmd name and description
if len(parts) == 2:
_option_list.append([parts[0], ":", new_parts[0], new_parts[1]])
# len will be 1 when there's continuation of multiline description in the next line
# check `blob_announce`'s `stream_hash` command
elif len(parts) == 1:
_option_list.append([None, None, None, parts[0]])
else:
print "Error: Ill formatted doc string for {}".format(method)
print "Error causing line: {}".format(line)
# tabulate to make the options look pretty
_options_docstr_no_indent = tabulate(_option_list, missingval="", tablefmt="plain")
# tabulate to make the options look pretty
_options_docstr = ""
for line in _options_docstr_no_indent.splitlines():
_options_docstr += INDENT + line + '\n'
return _options_docstr
def _cli_doc(obj):
docstr = (inspect.getdoc(obj) or '').strip()
try:
_usage_docstr, _docstr_after_options = docstr.split("Options:", 1)
_options_docstr, _returns_docstr = _docstr_after_options.split("Returns:", 1)
except(ValueError):
print "Error: Ill formatted doc string for {}".format(obj)
print "Please ensure that the docstring has all the three headings i.e. \"Usage:\""
print "\"Options:\" and \"Returns:\" exactly as specified, including the colon"
return "Error!"
try:
_options_docstr = _cli_tabulate_options(_options_docstr.strip(), obj)
except Exception as e:
print "Please make sure that the individual options are properly formatted"
print "It should be strictly of the format:"
print "--command_name=<command_name> : (type) desc"
print e.message
docstr = _usage_docstr + \
"\nOptions:\n" + \
_options_docstr + \
"\nReturns:" + \
_returns_docstr
return docstr
def _api_doc(obj):
docstr = (inspect.getdoc(obj) or '').strip()
try:
_desc, _docstr_after_desc = docstr.split("Usage:", 1)
_usage_docstr, _docstr_after_options = _docstr_after_desc.split("Options:", 1)
_options_docstr, _returns_docstr = _docstr_after_options.split("Returns:", 1)
except(ValueError):
print "Error: Ill formatted doc string for {}".format(obj)
print "Please ensure that the docstring has all the three headings i.e. \"Usage:\""
print "\"Options:\" and \"Returns:\" exactly as specified, including the colon"
return "Error!"
opt_matches = re.findall(OPT_CMD_REGEX, _usage_docstr)
reqd_matches = re.findall(REQD_CMD_REGEX, _usage_docstr)
try:
_options_docstr = _api_tabulate_options(_options_docstr.strip(), obj, reqd_matches, opt_matches)
except Exception as e:
print "Please make sure that the individual options are properly formatted"
print "It should be strictly of the format:"
print "--command_name=<command_name> : (type) desc"
print e.message
docstr = _desc + \
"Args:\n" + \
_options_docstr + \
"\nReturns:" + \
_returns_docstr
return docstr
def main():
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
build_dir = os.path.realpath(os.path.join(root_dir, DOCS_BUILD_DIR))
if not os.path.exists(build_dir):
os.makedirs(build_dir)
api_doc_path = os.path.join(build_dir, 'index.md')
cli_doc_path = os.path.join(build_dir, 'cli.md')
_api_docs = ''
_cli_docs = ''
for method_name in sorted(Daemon.callable_methods.keys()):
method = Daemon.callable_methods[method_name]
_api_docs += '## ' + method_name + "\n\n```text\n" + _api_doc(method) + "\n```\n\n"
_cli_docs += '## ' + method_name + "\n\n```text\n" + _cli_doc(method) + "\n```\n\n"
_api_docs = "# LBRY JSON-RPC API Documentation\n\n" + _api_docs
with open(api_doc_path, 'w+') as f:
f.write(_api_docs)
_cli_docs = "# LBRY JSON-RPC API Documentation\n\n" + _cli_docs
with open(cli_doc_path, 'w+') as f:
f.write(_cli_docs)
try:
subprocess.check_output("exec mkdocs build", cwd=root_dir, shell=True)
except subprocess.CalledProcessError as e:
print e.output
return 1
return 0
if __name__ == '__main__':
sys.exit(main())
-110
View File
@@ -1,110 +0,0 @@
import argparse
import hashlib
import json
import subprocess
import sys
import base58
from lbryum import SimpleConfig, Network
from lbryum.wallet import WalletStorage, Wallet
from lbryum.commands import known_commands, Commands
from lbryum import lbrycrd
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--wallet', help='path to lbryum wallet')
args = parser.parse_args()
ensureCliIsOnPathAndServerIsRunning()
wallet = getWallet(args.wallet)
addresses = wallet.addresses(True)
for addr in addresses[:-1]:
printBalance(wallet, addr)
saveAddr(wallet, addr)
# on the last one, rescan. Don't rescan early for sake of efficiency
addr = addresses[-1]
printBalance(wallet, addr)
saveAddr(wallet, addr, "true")
def ensureCliIsOnPathAndServerIsRunning():
try:
output = subprocess.check_output(['lbrycrd-cli', 'getinfo'])
except OSError:
print 'Failed to run: lbrycrd-cli needs to be on the PATH'
sys.exit(1)
except subprocess.CalledProcessError:
print 'Failed to run: could not connect to the lbrycrd server.'
print 'Make sure it is running and able to be connected to.'
print 'One way to do this is to run:'
print ' lbrycrdd -server -printtoconsole'
sys.exit(1)
def validateAddress(addr):
raw_output = subprocess.check_output(
['lbrycrd-cli', 'validateaddress', addr])
output = json.loads(raw_output)
if not output['isvalid']:
raise Exception('Address {} is not valid'.format(addr))
if not output['ismine']:
raise Exception('Address {} is not yours'.format(addr))
def printBalance(wallet, addr):
balance = getBalance(wallet, addr)
print 'Importing private key for %s with balance %s' % (addr, balance)
def getBalance(wallet, addr):
return sum(wallet.get_addr_balance(addr))
def getWallet(path=None):
if not path:
config = SimpleConfig()
path = config.get_wallet_path()
storage = WalletStorage(path)
if not storage.file_exists:
print "Failed to run: No wallet to migrate"
sys.exit(1)
return Wallet(storage)
def saveAddr(wallet, addr, rescan="false"):
keys = wallet.get_private_key(addr, None)
assert len(keys) == 1, 'Address {} has {} keys. Expected 1'.format(addr, len(keys))
key = keys[0]
# copied from lbrycrd.regenerate_key
b = lbrycrd.ASecretToSecret(key)
pkey = b[0:32]
is_compressed = lbrycrd.is_compressed(key)
wif = pkeyToWif(pkey, is_compressed)
subprocess.check_call(
['lbrycrd-cli', 'importprivkey', wif, "", rescan])
validateAddress(addr)
def pkeyToWif(pkey, compressed):
# Follow https://en.bitcoin.it/wiki/Wallet_import_format
# to convert from a private key to the wallet import format
prefix = '\x1c'
wif = prefix + pkey
if compressed:
wif += '\x01'
intermediate_checksum = hashlib.sha256(wif).digest()
checksum = hashlib.sha256(intermediate_checksum).digest()
wif = wif + checksum[:4]
return base58.b58encode(wif)
def wifToPkey(wif):
pkey = base58.b58decode(wif)
return pkey[1:-4]
if __name__ == '__main__':
sys.exit(main())
-94
View File
@@ -1,94 +0,0 @@
"""Reseed a file.
Given a file and a matching sd_blob,
re-chunk and encrypt the file, adding
the new blobs to the manager.
"""
import argparse
import binascii
import logging
import json
import os
import sys
from twisted.internet import defer
from twisted.internet import reactor
from twisted.protocols import basic
from lbrynet import conf
from lbrynet.core import BlobManager
from lbrynet.dht import hashannouncer
from lbrynet.core import log_support
from lbrynet.cryptstream import CryptStreamCreator
log = logging.getLogger('reseed_file')
def main():
conf.initialize_settings()
parser = argparse.ArgumentParser()
parser.add_argument('input_file')
parser.add_argument('sd_blob', help='a json file containing a key and the IVs')
args = parser.parse_args()
log_support.configure_console()
run(args)
reactor.run()
@defer.inlineCallbacks
def run(args):
try:
yield reseed_file(args.input_file, args.sd_blob)
except Exception as e:
log.exception('Failed to reseed')
finally:
reactor.stop()
@defer.inlineCallbacks
def reseed_file(input_file, sd_blob):
sd_blob = SdBlob.new_instance(sd_blob)
db_dir = conf.settings['data_dir']
blobfile_dir = os.path.join(db_dir, "blobfiles")
announcer = hashannouncer.DummyHashAnnouncer()
blob_manager = BlobManager.DiskBlobManager(announcer, blobfile_dir, db_dir)
yield blob_manager.setup()
creator = CryptStreamCreator.CryptStreamCreator(
blob_manager, None, sd_blob.key(), sd_blob.iv_generator())
file_sender = basic.FileSender()
with open(input_file) as f:
yield file_sender.beginFileTransfer(f, creator)
yield creator.stop()
for blob_info in sd_blob.blob_infos():
if 'blob_hash' not in blob_info:
# the last blob is always empty and without a hash
continue
blob = yield blob_manager.get_blob(blob_info['blob_hash'], True)
if not blob.verified:
print "Blob {} is not verified".format(blob)
class SdBlob(object):
def __init__(self, contents):
self.contents = contents
def key(self):
return binascii.unhexlify(self.contents['key'])
def iv_generator(self):
for blob_info in self.blob_infos():
yield binascii.unhexlify(blob_info['iv'])
def blob_infos(self):
return self.contents['blobs']
@classmethod
def new_instance(cls, filename):
with open(filename) as f:
return cls(json.load(f))
if __name__ == '__main__':
sys.exit(main())
-228
View File
@@ -1,228 +0,0 @@
import struct
import json
import logging
import argparse
import hashlib
from copy import deepcopy
from urllib import urlopen
from twisted.internet.epollreactor import install as install_epoll
install_epoll()
from twisted.internet import reactor, defer
from twisted.web import resource
from twisted.web.server import Site
from lbrynet import conf
from lbrynet.dht import constants
from lbrynet.dht.node import Node
from lbrynet.dht.error import TransportNotConnected
from lbrynet.core.log_support import configure_console, configure_twisted
from lbrynet.daemon.auth.server import AuthJSONRPCServer
# configure_twisted()
conf.initialize_settings()
configure_console()
lbrynet_handler = logging.getLogger("lbrynet").handlers[0]
log = logging.getLogger("dht router")
log.addHandler(lbrynet_handler)
log.setLevel(logging.INFO)
def node_id_supplier(seed="jack.lbry.tech"): # simple deterministic node id generator
h = hashlib.sha384()
h.update(seed)
while True:
next_id = h.digest()
yield next_id
h = hashlib.sha384()
h.update(seed)
h.update(next_id)
def get_external_ip():
response = json.loads(urlopen("https://api.lbry.io/ip").read())
if not response['success']:
raise ValueError("failed to get external ip")
return response['data']['ip']
def format_contact(contact):
return {
"node_id": contact.id.encode('hex'),
"address": contact.address,
"nodePort": contact.port,
"lastReplied": contact.lastReplied,
"lastRequested": contact.lastRequested,
"failedRPCs": contact.failedRPCs,
"lastFailed": None if not contact.failures else contact.failures[-1]
}
def format_datastore(node):
datastore = deepcopy(node._dataStore._dict)
result = {}
for key, values in datastore.items():
contacts = []
for (contact, value, last_published, originally_published, original_publisher_id) in values:
contact_dict = format_contact(contact)
contact_dict['peerPort'] = struct.unpack('>H', value[4:6])[0]
contact_dict['lastPublished'] = last_published
contact_dict['originallyPublished'] = originally_published
contact_dict['originalPublisherID'] = original_publisher_id.encode('hex')
contacts.append(contact_dict)
result[key.encode('hex')] = contacts
return result
class MultiSeedRPCServer(AuthJSONRPCServer):
def __init__(self, starting_node_port, nodes, rpc_port):
AuthJSONRPCServer.__init__(self, False)
self.port = None
self.rpc_port = rpc_port
self.external_ip = get_external_ip()
node_id_gen = node_id_supplier()
self._nodes = [Node(node_id=next(node_id_gen), udpPort=starting_node_port+i, externalIP=self.external_ip)
for i in range(nodes)]
self._own_addresses = [(self.external_ip, starting_node_port+i) for i in range(nodes)]
reactor.addSystemEventTrigger('after', 'startup', self.start)
@defer.inlineCallbacks
def start(self):
self.announced_startup = True
root = resource.Resource()
root.putChild('', self)
self.port = reactor.listenTCP(self.rpc_port, Site(root), interface='localhost')
log.info("starting %i nodes on %s, rpc available on localhost:%i", len(self._nodes), self.external_ip, self.rpc_port)
for node in self._nodes:
node.start_listening()
yield node._protocol._listening
for node1 in self._nodes:
for node2 in self._nodes:
if node1 is node2:
continue
try:
yield node1.addContact(node1.contact_manager.make_contact(node2.node_id, node2.externalIP,
node2.port, node1._protocol))
except TransportNotConnected:
pass
node1.safe_start_looping_call(node1._change_token_lc, constants.tokenSecretChangeInterval)
node1.safe_start_looping_call(node1._refresh_node_lc, constants.checkRefreshInterval)
node1._join_deferred = defer.succeed(True)
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
log.info("finished bootstrapping the network, running %i nodes", len(self._nodes))
@defer.inlineCallbacks
def stop(self):
yield self.port.stopListening()
yield defer.DeferredList([node.stop() for node in self._nodes])
def jsonrpc_get_node_ids(self):
return defer.succeed([node.node_id.encode('hex') for node in self._nodes])
def jsonrpc_node_datastore(self, node_id):
for node in self._nodes:
if node.node_id == node_id.decode('hex'):
return defer.succeed(format_datastore(node))
def jsonrpc_get_nodes_who_stored(self, blob_hash):
storing_nodes = {}
for node in self._nodes:
datastore = format_datastore(node)
if blob_hash in datastore:
storing_nodes[node.node_id.encode('hex')] = datastore[blob_hash]
return defer.succeed(storing_nodes)
def jsonrpc_node_routing_table(self, node_id):
def format_bucket(bucket):
return {
"contacts": [format_contact(contact) for contact in bucket._contacts],
"lastAccessed": bucket.lastAccessed
}
def format_routing(node):
return {
i: format_bucket(bucket) for i, bucket in enumerate(node._routingTable._buckets)
}
for node in self._nodes:
if node.node_id == node_id.decode('hex'):
return defer.succeed(format_routing(node))
def jsonrpc_restart_node(self, node_id):
for node in self._nodes:
if node.node_id == node_id.decode('hex'):
d = node.stop()
d.addCallback(lambda _: node.start(self._own_addresses))
return d
@defer.inlineCallbacks
def jsonrpc_local_node_rpc(self, from_node, query, args=()):
def format_result(response):
if isinstance(response, list):
return [[node_id.encode('hex'), address, port] for (node_id, address, port) in response]
if isinstance(response, dict):
return {'token': response['token'].encode('hex'), 'contacts': format_result(response['contacts'])}
return response
for node in self._nodes:
if node.node_id == from_node.decode('hex'):
fn = getattr(node, query)
self_contact = node.contact_manager.make_contact(node.node_id, node.externalIP, node.port, node._protocol)
if args:
args = (str(arg) if isinstance(arg, (str, unicode)) else int(arg) for arg in args)
result = yield fn(self_contact, *args)
else:
result = yield fn()
# print "result: %s" % result
defer.returnValue(format_result(result))
@defer.inlineCallbacks
def jsonrpc_node_rpc(self, from_node, to_node, query, args=()):
def format_result(response):
if isinstance(response, list):
return [[node_id.encode('hex'), address, port] for (node_id, address, port) in response]
if isinstance(response, dict):
return {'token': response['token'].encode('hex'), 'contacts': format_result(response['contacts'])}
return response
for node in self._nodes:
if node.node_id == from_node.decode('hex'):
remote = node._routingTable.getContact(to_node.decode('hex'))
fn = getattr(remote, query)
if args:
args = (str(arg).decode('hex') for arg in args)
result = yield fn(*args)
else:
result = yield fn()
defer.returnValue(format_result(result))
@defer.inlineCallbacks
def jsonrpc_get_nodes_who_know(self, ip_address):
nodes = []
for node_id in [n.node_id.encode('hex') for n in self._nodes]:
routing_info = yield self.jsonrpc_node_routing_table(node_id=node_id)
for index, bucket in routing_info.items():
if ip_address in map(lambda c: c['address'], bucket['contacts']):
nodes.append(node_id)
break
defer.returnValue(nodes)
def jsonrpc_node_status(self):
return defer.succeed({
node.node_id.encode('hex'): node._join_deferred is not None and node._join_deferred.called
for node in self._nodes
})
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--rpc_port', default=5280)
parser.add_argument('--starting_port', default=4455)
parser.add_argument('--nodes', default=32)
args = parser.parse_args()
MultiSeedRPCServer(int(args.starting_port), int(args.nodes), int(args.rpc_port))
reactor.run()
if __name__ == "__main__":
main()
+1 -1
View File
@@ -155,7 +155,7 @@ class TestTransfer(unittest.TestCase):
@defer.inlineCallbacks
def test_lbry_transfer(self):
sd_blob = yield download_sd_blob(
self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.prm, self.wallet
self.conf, self.sd_hash, self.blob_manager, self.peer_finder, self.rate_limiter, self.prm, self.wallet
)
metadata = yield self.sd_identifier.get_metadata_for_sd_blob(sd_blob)
downloader = yield metadata.factories[0].make_downloader(
+4 -3
View File
@@ -4,6 +4,7 @@ import tempfile
from hashlib import md5
from twisted.trial.unittest import TestCase
from twisted.internet import defer, threads
from lbrynet.conf import Config
from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.p2p.StreamDescriptor import get_sd_info
@@ -31,12 +32,12 @@ class TestStreamify(TestCase):
@defer.inlineCallbacks
def setUp(self):
mocks.mock_conf_settings(self)
self.session = None
self.lbry_file_manager = None
self.is_generous = True
self.db_dir = tempfile.mkdtemp()
self.blob_dir = os.path.join(self.db_dir, "blobfiles")
conf = Config(data_dir=self.blob_dir)
os.mkdir(self.blob_dir)
self.dht_node = FakeNode()
self.wallet = FakeWallet()
@@ -44,11 +45,11 @@ class TestStreamify(TestCase):
self.peer_finder = FakePeerFinder(5553, self.peer_manager, 2)
self.rate_limiter = DummyRateLimiter()
self.sd_identifier = StreamDescriptorIdentifier()
self.storage = SQLiteStorage(':memory:')
self.storage = SQLiteStorage(conf, ':memory:')
self.blob_manager = DiskBlobManager(self.blob_dir, self.storage, self.dht_node._dataStore)
self.prm = OnlyFreePaymentsManager()
self.lbry_file_manager = EncryptedFileManager(
self.peer_finder, self.rate_limiter, self.blob_manager, self.wallet, self.prm, self.storage,
conf, self.peer_finder, self.rate_limiter, self.blob_manager, self.wallet, self.prm, self.storage,
self.sd_identifier
)
yield f2d(self.storage.open())
+9 -9
View File
@@ -1,9 +1,8 @@
import contextlib
from io import StringIO
from unittest import skip
from torba.testcase import AsyncioTestCase
from lbrynet import conf
from lbrynet.conf import Config
from lbrynet.extras import cli
from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, \
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, \
@@ -37,13 +36,14 @@ class CLIIntegrationTest(AsyncioTestCase):
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT,
RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT
]
conf.initialize_settings(load_conf_file=False)
conf.settings['api_port'] = 5299
conf.settings['components_to_skip'] = skip
conf.settings.initialize_post_conf_load()
conf = Config()
conf.data_dir = '/tmp'
conf.share_usage_data = False
conf.api = 'localhost:5299'
conf.components_to_skip = skip
Daemon.component_attributes = {}
self.daemon = Daemon(analytics_manager=FakeAnalytics())
await self.daemon.start_listening()
self.daemon = Daemon(conf)
await self.daemon.start()
async def asyncTearDown(self):
await self.daemon.shutdown()
@@ -51,6 +51,6 @@ class CLIIntegrationTest(AsyncioTestCase):
def test_cli_status_command_with_auth(self):
actual_output = StringIO()
with contextlib.redirect_stdout(actual_output):
cli.main(["status"])
cli.main(["--api", "localhost:5299", "status"])
actual_output = actual_output.getvalue()
self.assertIn("connection_status", actual_output)
+44 -23
View File
@@ -2,6 +2,7 @@ import sys
import json
import tempfile
import logging
import time
from binascii import unhexlify
import twisted.internet
@@ -16,7 +17,7 @@ from torba.testcase import IntegrationTestCase
import lbrynet.schema
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
from lbrynet import conf as lbry_conf
from lbrynet.conf import Config
from lbrynet.extras.daemon.Daemon import Daemon, jsonrpc_dumps_pretty
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.daemon.Components import WalletComponent
@@ -63,19 +64,16 @@ class CommandTestCase(IntegrationTestCase):
logging.getLogger('lbrynet.p2p').setLevel(self.VERBOSITY)
logging.getLogger('lbrynet.daemon').setLevel(self.VERBOSITY)
lbry_conf.settings = None
lbry_conf.initialize_settings(
load_conf_file=False,
data_dir=self.wallet_node.data_path,
wallet_dir=self.wallet_node.data_path,
download_dir=self.wallet_node.data_path
)
lbry_conf.settings['use_upnp'] = False
lbry_conf.settings['reflect_uploads'] = False
lbry_conf.settings['blockchain_name'] = 'lbrycrd_regtest'
lbry_conf.settings['lbryum_servers'] = [('localhost', 50001)]
lbry_conf.settings['known_dht_nodes'] = []
lbry_conf.settings.node_id = None
conf = Config()
conf.data_dir = self.wallet_node.data_path
conf.wallet_dir = self.wallet_node.data_path
conf.download_dir = self.wallet_node.data_path
conf.share_usage_data = False
conf.use_upnp = False
conf.reflect_uploads = False
conf.blockchain_name = 'lbrycrd_regtest'
conf.lbryum_servers = [('localhost', 50001)]
conf.known_dht_nodes = []
await self.account.ensure_address_gap()
address = (await self.account.receiving.get_addresses(limit=1, only_usable=True))[0]
@@ -89,14 +87,12 @@ class CommandTestCase(IntegrationTestCase):
self.wallet_component._running = True
return self.wallet_component
skip = [
conf.components_to_skip = [
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
]
analytics_manager = FakeAnalytics()
self.daemon = Daemon(analytics_manager, ComponentManager(
analytics_manager=analytics_manager,
skip_components=skip, wallet=wallet_maker
self.daemon = Daemon(conf, ComponentManager(
conf, skip_components=conf.components_to_skip, wallet=wallet_maker
))
await self.daemon.setup()
self.daemon.wallet_manager = self.wallet_component.wallet_manager
@@ -247,7 +243,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
# After abandoning he just waits for his LBCs to be returned to his account
await self.generate(5)
result = await self.daemon.jsonrpc_account_balance()
self.assertEqual(result, '8.9693585')
self.assertEqual(result, '8.969363')
# Amidst all this Chris receives a call from his friend Ramsey
# who says that it is of utmost urgency that Chris transfer him
@@ -264,7 +260,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
await self.generate(5)
result = await self.daemon.jsonrpc_account_balance()
# Chris' balance was correct
self.assertEqual(result, '7.9692345')
self.assertEqual(result, '7.969239')
# Ramsey too assured him that he had received the 1 LBC and thanks him
result = await self.daemon.jsonrpc_account_balance(ramsey_account_id)
@@ -423,12 +419,12 @@ class ClaimManagement(CommandTestCase):
VERBOSITY = logging.WARN
async def make_claim(self, name='hovercraft', amount='1.0', data=b'hi!', channel_name=None, confirm=True):
async def make_claim(self, name='hovercraft', amount='1.0', data=b'hi!', channel_name=None, confirm=True, **kwargs):
with tempfile.NamedTemporaryFile() as file:
file.write(data)
file.flush()
claim = await self.out(self.daemon.jsonrpc_publish(
name, amount, file_path=file.name, channel_name=channel_name
name, amount, file_path=file.name, channel_name=channel_name, **kwargs
))
self.assertTrue(claim['success'])
if confirm:
@@ -617,6 +613,31 @@ class ClaimManagement(CommandTestCase):
)
self.assertEqual(not_a_claim, 'claim not found')
async def test_release_time_create_update(self):
channel = await self.out(self.daemon.jsonrpc_channel_new('@rickroll-news', "1.0"))
self.assertTrue(channel['success'])
await self.confirm_tx(channel['tx']['txid'])
release_time = int(time.time())
claim = await self.make_claim(name='today', channel_name='@rickroll-news', release_time=release_time)
self.assertTrue(claim['success'])
uri = 'lbry://@rickroll-news/today'
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
self.assertEqual(response[uri]['claim']['value']['stream']['metadata']['releaseTime'], release_time)
self.assertTrue(response[uri]['claim']['signature_is_valid'])
# moar news! stop the machines! update today's edition with extra metadata!
release_time = int(time.time())
self.assertTrue(claim['success'])
metadata = response[uri]['claim']['value']['stream']['metadata']
metadata['title'] = "[YOU WONT BELIEVE IT!] Hamster tries to climb a tree"
metadata['description'] = "Never gonna give you up; Never gonna let you down"
metadata['releaseTime'] = release_time
await self.make_claim(amount='0.0001', name='today', channel_name='@rickroll-news', metadata=metadata)
response = await self.out(self.daemon.jsonrpc_resolve(uri=uri))
self.assertEqual(response[uri]['claim']['value']['stream']['metadata'], metadata)
self.assertTrue(response[uri]['claim']['signature_is_valid'])
async def test_abandoned_channel_with_signed_claims(self):
channel = await self.out(self.daemon.jsonrpc_channel_new('@abc', "1.0"))
self.assertTrue(channel['success'])
@@ -101,34 +101,6 @@ class BasicTransactionTest(IntegrationTestCase):
self.assertEqual('URI lbry://404 cannot be resolved', response['lbry://404']['error'])
self.assertEqual('URI lbry://@404 cannot be resolved', response['lbry://@404']['error'])
async def test_new_signature_model(self):
address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True)
sendtxid1 = await self.blockchain.send_to_address(address1, 5)
sendtxid2 = await self.blockchain.send_to_address(address2, 5)
await self.blockchain.generate(1)
await asyncio.wait([
self.on_transaction_id(sendtxid1),
self.on_transaction_id(sendtxid2)
])
self.assertEqual(d2l(await self.account.get_balance()), '10.0')
cert, key = generate_certificate()
cert_tx = await Transaction.claim('@bar', cert, l2d('1.0'), address1, [self.account], self.account)
claim = ClaimDict.load_dict(example_claim_dict)
claim = claim.sign(key, address1, cert_tx.outputs[0].claim_id, name='foo', curve=SECP256k1, force_detached=True)
claim_tx = await Transaction.claim('foo', claim, l2d('1.0'), address1, [self.account], self.account)
await self.broadcast(cert_tx)
await self.broadcast(claim_tx)
await self.ledger.wait(claim_tx)
await self.blockchain.generate(1)
await self.ledger.wait(claim_tx)
response = await self.ledger.resolve(0, 10, 'lbry://@bar/foo')
self.assertIn('lbry://@bar/foo', response)
self.assertIn('claim', response['lbry://@bar/foo'])
async def test_new_signature_model_from_unserializable_claim(self):
address1, address2 = await self.account.receiving.get_addresses(limit=2, only_usable=True)
sendtxid1 = await self.blockchain.send_to_address(address1, 5)
+2 -24
View File
@@ -8,13 +8,13 @@ from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import serialization
from twisted.internet import defer
from twisted.python.failure import Failure
from lbrynet import conf
from lbrynet.conf import Config
from lbrynet.p2p.client.ClientRequest import ClientRequest
from lbrynet.p2p.Error import RequestCanceledError
from lbrynet.p2p import BlobAvailability
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
from lbrynet.dht.node import Node as RealNode
from lbrynet.extras.daemon import ExchangeRateManager as ERM
from lbrynet.extras.daemon import exchange_rate_manager as ERM
KB = 2**10
PUBLIC_EXPONENT = 65537 # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html
@@ -472,25 +472,3 @@ create_stream_sd_file = {
'stream_hash': '6d27fbe10c86d81aacfb897c7a426d0a2214f5a299455a6d'
'315c0f998c4b3545c2dc60906122d94653c23b1898229e3f'
}
def mock_conf_settings(obj, settings={}):
conf.settings = None
settings.setdefault('download_mirrors', [])
conf.initialize_settings(False)
original_settings = conf.settings
conf.settings = conf.Config(conf.FIXED_SETTINGS, conf.ADJUSTABLE_SETTINGS)
conf.settings['data_dir'] = settings.get('data_dir') or conf.settings.data_dir \
or conf.settings.default_data_dir
conf.settings['download_directory'] = settings.get('download_directory') or conf.settings.download_dir \
or conf.settings.default_download_dir
conf.settings['wallet_dir'] = settings.get('wallet_dir') or conf.settings.wallet_dir or \
conf.settings.default_wallet_dir
conf.settings.installation_id = conf.settings.get_installation_id()
conf.settings.node_id = conf.settings.get_node_id()
conf.settings.update(settings)
def _reset_settings():
conf.settings = original_settings
obj.addCleanup(_reset_settings)
@@ -2,19 +2,18 @@ import asyncio
from unittest import TestCase
from torba.testcase import AdvanceTimeTestCase
from tests import mocks
from lbrynet.conf import Config
from lbrynet.extras.daemon.ComponentManager import ComponentManager
from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT
from lbrynet.extras.daemon.Components import HASH_ANNOUNCER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT
from lbrynet.extras.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
from lbrynet.extras.daemon.Components import RATE_LIMITER_COMPONENT, HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT
from lbrynet.extras.daemon import Components
from tests import mocks
class TestComponentManager(TestCase):
def setUp(self):
mocks.mock_conf_settings(self)
self.default_components_sort = [
[
Components.HeadersComponent,
@@ -38,7 +37,7 @@ class TestComponentManager(TestCase):
Components.ReflectorComponent
]
]
self.component_manager = ComponentManager()
self.component_manager = ComponentManager(Config())
def tearDown(self):
pass
@@ -62,9 +61,6 @@ class TestComponentManager(TestCase):
class TestComponentManagerOverrides(TestCase):
def setUp(self):
mocks.mock_conf_settings(self)
def test_init_with_overrides(self):
class FakeWallet:
component_name = "wallet"
@@ -77,7 +73,7 @@ class TestComponentManagerOverrides(TestCase):
def component(self):
return self
new_component_manager = ComponentManager(wallet=FakeWallet)
new_component_manager = ComponentManager(Config(), wallet=FakeWallet)
fake_wallet = new_component_manager.get_component("wallet")
# wallet should be an instance of FakeWallet and not WalletComponent from Components.py
self.assertIsInstance(fake_wallet, FakeWallet)
@@ -89,14 +85,14 @@ class TestComponentManagerOverrides(TestCase):
depends_on = []
with self.assertRaises(SyntaxError):
ComponentManager(randomComponent=FakeRandomComponent)
ComponentManager(Config(), randomComponent=FakeRandomComponent)
class TestComponentManagerProperStart(AdvanceTimeTestCase):
def setUp(self):
mocks.mock_conf_settings(self)
self.component_manager = ComponentManager(
Config(),
skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT,
HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT, RATE_LIMITER_COMPONENT,
@@ -11,6 +11,7 @@ from lbrynet.p2p.RateLimiter import RateLimiter
from lbrynet.p2p.Peer import Peer
from lbrynet.p2p.Error import NoResponseError
from lbrynet.extras.daemon.PeerManager import PeerManager
from lbrynet.conf import Config
PEER_PORT = 5551
LOCAL_HOST = '127.0.0.1'
@@ -118,10 +119,11 @@ class TestIntegrationConnectionManager(TestCase):
def setUp(self):
conf.initialize_settings(False)
conf = Config()
self.TEST_PEER = Peer(LOCAL_HOST, PEER_PORT)
self.downloader = MocDownloader()
self.downloader.conf = conf
self.rate_limiter = RateLimiter()
self.primary_request_creator = MocRequestCreator([self.TEST_PEER])
self.clock = task.Clock()
@@ -7,15 +7,17 @@ from twisted.trial import unittest
from lbrynet.p2p import Peer
from lbrynet.p2p.PaymentRateManager import NegotiatedPaymentRateManager, BasePaymentRateManager
from tests.mocks import BlobAvailabilityTracker as DummyBlobAvailabilityTracker, mock_conf_settings
from lbrynet.conf import Config
from tests.mocks import BlobAvailabilityTracker as DummyBlobAvailabilityTracker
class TestBlobRequestHandlerQueries(unittest.TestCase):
def setUp(self):
mock_conf_settings(self)
conf = Config()
self.blob_manager = mock.Mock()
self.payment_rate_manager = NegotiatedPaymentRateManager(
BasePaymentRateManager(0.001), DummyBlobAvailabilityTracker())
BasePaymentRateManager(0.001, conf.min_info_rate), DummyBlobAvailabilityTracker(), conf.is_generous_host
)
from lbrynet.p2p.server import BlobRequestHandler
self.handler = BlobRequestHandler.BlobRequestHandler(
self.blob_manager, None, self.payment_rate_manager, None)
+2 -3
View File
@@ -11,18 +11,17 @@ from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.extras.compat import f2d
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.p2p.Peer import Peer
from lbrynet import conf
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbrynet.conf import Config
class BlobManagerTest(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
conf.initialize_settings(False)
self.blob_dir = tempfile.mkdtemp()
self.db_dir = tempfile.mkdtemp()
self.bm = DiskBlobManager(self.blob_dir, SQLiteStorage(':memory:'))
self.bm = DiskBlobManager(self.blob_dir, SQLiteStorage(Config(data_dir=self.blob_dir), ':memory:'))
self.peer = Peer('somehost', 22)
yield f2d(self.bm.storage.open())
+7 -5
View File
@@ -19,7 +19,9 @@ class HTTPBlobDownloaderTest(unittest.TestCase):
self.blob_manager.get_blob.side_effect = lambda _: self.blob
self.response = MagicMock(code=200, length=400)
self.client.get.side_effect = lambda uri: defer.succeed(self.response)
self.downloader = HTTPBlobDownloader(self.blob_manager, [self.blob_hash], ['server1'], self.client, retry=False)
self.downloader = HTTPBlobDownloader(
self.blob_manager, [self.blob_hash], [('server1', 80)], self.client, retry=False
)
self.downloader.interval = 0
def tearDown(self):
@@ -31,7 +33,7 @@ class HTTPBlobDownloaderTest(unittest.TestCase):
self.client.collect.side_effect = collect
yield self.downloader.start()
self.blob_manager.get_blob.assert_called_with(self.blob_hash)
self.client.get.assert_called_with('http://{}/{}'.format('server1', self.blob_hash))
self.client.get.assert_called_with(f'http://server1:80/{self.blob_hash}')
self.client.collect.assert_called()
self.assertEqual(self.blob.get_length(), self.response.length)
self.assertTrue(self.blob.get_is_verified())
@@ -50,7 +52,7 @@ class HTTPBlobDownloaderTest(unittest.TestCase):
self.client.collect.side_effect = lambda response, write: defer.fail(IOError('I/O operation on closed file'))
yield self.downloader.start()
self.blob_manager.get_blob.assert_called_with(self.blob_hash)
self.client.get.assert_called_with('http://{}/{}'.format('server1', self.blob_hash))
self.client.get.assert_called_with(f'http://server1:80/{self.blob_hash}')
self.client.collect.assert_called()
self.assertEqual(self.blob.get_length(), self.response.length)
self.assertEqual(self.blob.writers, {})
@@ -70,7 +72,7 @@ class HTTPBlobDownloaderTest(unittest.TestCase):
self.response.code = 404
yield self.downloader.start()
self.blob_manager.get_blob.assert_called_with(self.blob_hash)
self.client.get.assert_called_with('http://{}/{}'.format('server1', self.blob_hash))
self.client.get.assert_called_with(f'http://server1:80/{self.blob_hash}')
self.client.collect.assert_not_called()
self.assertFalse(self.blob.get_is_verified())
self.assertEqual(self.blob.writers, {})
@@ -80,7 +82,7 @@ class HTTPBlobDownloaderTest(unittest.TestCase):
self.downloader.start() # hangs if yielded, as intended, to simulate a long ongoing write while we call stop
self.downloader.stop()
self.blob_manager.get_blob.assert_called_with(self.blob_hash)
self.client.get.assert_called_with('http://{}/{}'.format('server1', self.blob_hash))
self.client.get.assert_called_with(f'http://server1:80/{self.blob_hash}')
self.client.collect.assert_called()
self.assertEqual(self.blob.get_length(), self.response.length)
self.assertFalse(self.blob.get_is_verified())
+19 -11
View File
@@ -7,8 +7,8 @@ from twisted.trial import unittest
from lbrynet.p2p.PaymentRateManager import NegotiatedPaymentRateManager, BasePaymentRateManager
from lbrynet.p2p.Strategy import BasicAvailabilityWeightedStrategy
from lbrynet.p2p.Offer import Offer
from tests.mocks\
import BlobAvailabilityTracker as DummyBlobAvailabilityTracker, mock_conf_settings
from lbrynet.conf import Config
from tests.mocks import BlobAvailabilityTracker as DummyBlobAvailabilityTracker
MAX_NEGOTIATION_TURNS = 10
random.seed(12345)
@@ -52,14 +52,16 @@ def calculate_negotation_turns(client_base, host_base, host_is_generous=True,
client = mock.Mock()
client.host = "1.2.3.5"
client_base_prm = BasePaymentRateManager(client_base)
conf = Config()
client_base_prm = BasePaymentRateManager(client_base, conf.min_info_rate)
client_prm = NegotiatedPaymentRateManager(client_base_prm,
DummyBlobAvailabilityTracker(),
generous=client_is_generous)
host_base_prm = BasePaymentRateManager(host_base)
client_is_generous)
host_base_prm = BasePaymentRateManager(host_base, conf.min_info_rate)
host_prm = NegotiatedPaymentRateManager(host_base_prm,
DummyBlobAvailabilityTracker(),
generous=host_is_generous)
host_is_generous)
blobs_to_query = get_random_sample(blobs)
accepted = False
turns = 0
@@ -72,11 +74,12 @@ def calculate_negotation_turns(client_base, host_base, host_is_generous=True,
class AvailabilityWeightedStrategyTests(unittest.TestCase):
def setUp(self):
mock_conf_settings(self)
def test_first_offer_is_zero_and_second_is_not_if_offer_not_accepted(self):
strategy = BasicAvailabilityWeightedStrategy(DummyBlobAvailabilityTracker())
conf = Config()
strategy = BasicAvailabilityWeightedStrategy(
DummyBlobAvailabilityTracker(), conf.data_rate, conf.is_generous_host
)
peer = "1.1.1.1"
blobs = strategy.price_model.blob_tracker.availability.keys()
@@ -88,8 +91,13 @@ class AvailabilityWeightedStrategyTests(unittest.TestCase):
self.assertNotEqual(offer2.rate, 0.0)
def test_accept_zero_and_persist_if_accepted(self):
host_strategy = BasicAvailabilityWeightedStrategy(DummyBlobAvailabilityTracker())
client_strategy = BasicAvailabilityWeightedStrategy(DummyBlobAvailabilityTracker())
conf = Config()
host_strategy = BasicAvailabilityWeightedStrategy(
DummyBlobAvailabilityTracker(), conf.data_rate, conf.is_generous_host
)
client_strategy = BasicAvailabilityWeightedStrategy(
DummyBlobAvailabilityTracker(), conf.data_rate, conf.is_generous_host
)
client = "1.1.1.1"
host = "1.1.1.2"
-5
View File
@@ -3,8 +3,6 @@ from twisted.internet import defer
from lbrynet.blob import CryptBlob
from lbrynet.blob.blob_file import MAX_BLOB_SIZE
from tests.mocks import mock_conf_settings
from cryptography.hazmat.primitives.ciphers.algorithms import AES
import random
import string
@@ -39,9 +37,6 @@ def random_string(length):
class TestCryptBlob(unittest.TestCase):
def setUp(self):
mock_conf_settings(self)
@defer.inlineCallbacks
def _test_encrypt_decrypt(self, size_of_data):
+2 -3
View File
@@ -5,7 +5,7 @@ import logging
from copy import deepcopy
from twisted.internet import defer
from twisted.trial import unittest
from lbrynet import conf
from lbrynet.conf import Config
from lbrynet.extras.compat import f2d
from lbrynet.extras.daemon.storage import SQLiteStorage, open_file_for_writing
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
@@ -84,9 +84,8 @@ class StorageTest(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
conf.initialize_settings(False)
self.db_dir = tempfile.mkdtemp()
self.storage = SQLiteStorage(':memory:')
self.storage = SQLiteStorage(Config(data_dir=self.db_dir), ':memory:')
yield f2d(self.storage.open())
@defer.inlineCallbacks
+3 -3
View File
@@ -1,9 +1,9 @@
from twisted.trial import unittest
from twisted.internet import defer, task
from lbrynet import utils
from lbrynet.conf import Config
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
from tests.test_utils import random_lbry_hash
from tests.mocks import mock_conf_settings
class MocDHTNode:
@@ -38,7 +38,7 @@ class MocStorage:
class DHTHashAnnouncerTest(unittest.TestCase):
def setUp(self):
mock_conf_settings(self)
conf = Config()
self.num_blobs = 10
self.blobs_to_announce = []
for i in range(0, self.num_blobs):
@@ -47,7 +47,7 @@ class DHTHashAnnouncerTest(unittest.TestCase):
self.clock = self.dht_node.clock
utils.call_later = self.clock.callLater
self.storage = MocStorage(self.blobs_to_announce)
self.announcer = DHTHashAnnouncer(self.dht_node, self.storage)
self.announcer = DHTHashAnnouncer(conf, self.dht_node, self.storage)
@defer.inlineCallbacks
def test_immediate_announce(self):
@@ -3,6 +3,7 @@ from twisted.trial import unittest
from twisted.internet import defer
from cryptography.hazmat.primitives.ciphers.algorithms import AES
from lbrynet.conf import Config
from lbrynet.extras.compat import f2d
from lbrynet.extras.daemon.PeerManager import PeerManager
from lbrynet.p2p.StreamDescriptor import get_sd_info, BlobStreamDescriptorReader
@@ -38,18 +39,20 @@ class CreateEncryptedFileTest(unittest.TestCase):
timeout = 5
def setUp(self):
mocks.mock_conf_settings(self)
self.tmp_db_dir, self.tmp_blob_dir = mk_db_and_blob_dir()
conf = Config(data_dir=self.tmp_blob_dir)
self.wallet = FakeWallet()
self.peer_manager = PeerManager()
self.peer_finder = FakePeerFinder(5553, self.peer_manager, 2)
self.rate_limiter = DummyRateLimiter()
self.sd_identifier = StreamDescriptorIdentifier()
self.storage = SQLiteStorage(':memory:')
self.storage = SQLiteStorage(conf, ':memory:')
self.blob_manager = DiskBlobManager(self.tmp_blob_dir, self.storage)
self.prm = OnlyFreePaymentsManager()
self.lbry_file_manager = EncryptedFileManager(self.peer_finder, self.rate_limiter, self.blob_manager,
self.wallet, self.prm, self.storage, self.sd_identifier)
self.lbry_file_manager = EncryptedFileManager(
conf, self.peer_finder, self.rate_limiter, self.blob_manager,
self.wallet, self.prm, self.storage, self.sd_identifier
)
d = f2d(self.storage.open())
d.addCallback(lambda _: f2d(self.lbry_file_manager.setup()))
return d
Whitespace-only changes.
@@ -1,107 +0,0 @@
from unittest import mock, skip
from twisted.internet import reactor
from twisted.trial import unittest
from lbrynet import conf
#from lbrynet.extras.daemon.auth import server
from tests.mocks import mock_conf_settings
@skip
class AuthJSONRPCServerTest(unittest.TestCase):
# TODO: move to using a base class for tests
# and add useful general utilities like this
# onto it.
def setUp(self):
conf.initialize_settings(False)
self.server = server.AuthJSONRPCServer(True, use_authentication=False)
def test_listen_auth_https(self):
self.server._use_https = True
self.server._use_authentication = True
factory = self.server.get_server_factory()
listening_port = reactor.listenSSL(
conf.settings['api_port'], factory, factory.options, interface="localhost"
)
listening_port.stopListening()
def test_listen_no_auth_https(self):
self.server._use_https = True
self.server._use_authentication = False
factory = self.server.get_server_factory()
listening_port = reactor.listenSSL(
conf.settings['api_port'], factory, factory.options, interface="localhost"
)
listening_port.stopListening()
def test_listen_auth_http(self):
self.server._use_https = False
self.server._use_authentication = True
factory = self.server.get_server_factory()
listening_port = reactor.listenTCP(
conf.settings['api_port'], factory, interface="localhost"
)
listening_port.stopListening()
def test_listen_no_auth_http(self):
self.server._use_https = False
self.server._use_authentication = False
factory = self.server.get_server_factory()
listening_port = reactor.listenTCP(
conf.settings['api_port'], factory, interface="localhost"
)
listening_port.stopListening()
def test_get_server_port(self):
self.assertSequenceEqual(
('example.com', 80), self.server.get_server_port('http://example.com'))
self.assertSequenceEqual(
('example.com', 1234), self.server.get_server_port('http://example.com:1234'))
def test_foreign_origin_is_rejected(self):
mock_conf_settings(self) # have to call this to generate Config mock
request = mock.Mock(['getHeader'])
request.getHeader = mock.Mock(return_value='http://example.com')
self.assertFalse(self.server._check_header_source(request, 'Origin'))
def test_wrong_port_is_rejected(self):
mock_conf_settings(self, {'api_port': 1234})
request = mock.Mock(['getHeader'])
request.getHeader = mock.Mock(return_value='http://localhost:9999')
self.assertFalse(self.server._check_header_source(request, 'Origin'))
def test_matching_origin_is_allowed(self):
mock_conf_settings(self, {'api_host': 'example.com', 'api_port': 1234})
request = mock.Mock(['getHeader'])
request.getHeader = mock.Mock(return_value='http://example.com:1234')
self.assertTrue(self.server._check_header_source(request, 'Origin'))
def test_any_origin_is_allowed(self):
mock_conf_settings(self, {'api_host': '0.0.0.0', 'api_port': 80})
request = mock.Mock(['getHeader'])
request.getHeader = mock.Mock(return_value='http://example.com')
self.assertTrue(self.server._check_header_source(request, 'Origin'))
request = mock.Mock(['getHeader'])
request.getHeader = mock.Mock(return_value='http://another-example.com')
self.assertTrue(self.server._check_header_source(request, 'Origin'))
def test_matching_referer_is_allowed(self):
mock_conf_settings(self, {'api_host': 'the_api', 'api_port': 1111})
request = mock.Mock(['getHeader'])
request.getHeader = mock.Mock(return_value='http://the_api:1111?settings')
self.assertTrue(self.server._check_header_source(request, 'Referer'))
request.getHeader.assert_called_with('Referer')
def test_request_is_allowed_when_matching_allowed_origin_setting(self):
mock_conf_settings(self, {'allowed_origin': 'http://example.com:1234'})
request = mock.Mock(['getHeader'])
request.getHeader = mock.Mock(return_value='http://example.com:1234')
self.assertTrue(self.server._check_header_source(request, 'Origin'))
def test_request_is_rejected_when_not_matching_allowed_origin_setting(self):
mock_conf_settings(self, {'allowed_origin': 'http://example.com:1234'})
request = mock.Mock(['getHeader'])
# note the ports don't match
request.getHeader = mock.Mock(return_value='http://example.com:1235')
self.assertFalse(self.server._check_header_source(request, 'Origin'))
+21 -24
View File
@@ -4,7 +4,6 @@ import json
from twisted.internet import defer
from twisted.trial import unittest
from lbrynet import conf
from lbrynet.extras.compat import f2d
from lbrynet.schema.decode import smart_decode
from lbrynet.extras.daemon.storage import SQLiteStorage
@@ -20,28 +19,30 @@ from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
from lbrynet.extras.wallet import LbryWalletManager
from torba.client.wallet import Wallet
from lbrynet.conf import Config
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
from tests import test_utils
from tests.mocks import mock_conf_settings, FakeNetwork, FakeFileManager
from tests.mocks import FakeNetwork, FakeFileManager
from tests.mocks import ExchangeRateManager as DummyExchangeRateManager
from tests.mocks import BTCLBCFeed, USDBTCFeed
from tests.test_utils import is_android
def get_test_daemon(data_rate=None, generous=True, with_fee=False):
if data_rate is None:
data_rate = conf.ADJUSTABLE_SETTINGS['data_rate'][1]
def get_test_daemon(conf: Config, with_fee=False):
conf.data_dir = '/tmp'
rates = {
'BTCLBC': {'spot': 3.0, 'ts': test_utils.DEFAULT_ISO_TIME + 1},
'USDBTC': {'spot': 2.0, 'ts': test_utils.DEFAULT_ISO_TIME + 2}
}
component_manager = ComponentManager(
skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, UPNP_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, HASH_ANNOUNCER_COMPONENT,
EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
HEADERS_COMPONENT, RATE_LIMITER_COMPONENT],
conf, skip_components=[
DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, UPNP_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, HASH_ANNOUNCER_COMPONENT,
EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
HEADERS_COMPONENT, RATE_LIMITER_COMPONENT],
file_manager=FakeFileManager
)
daemon = LBRYDaemon(component_manager=component_manager)
daemon = LBRYDaemon(conf, component_manager=component_manager)
daemon.payment_rate_manager = OnlyFreePaymentsManager()
daemon.wallet_manager = mock.Mock(spec=LbryWalletManager)
daemon.wallet_manager.wallet = mock.Mock(spec=Wallet)
@@ -80,24 +81,23 @@ def get_test_daemon(data_rate=None, generous=True, with_fee=False):
class TestCostEst(unittest.TestCase):
def setUp(self):
mock_conf_settings(self)
test_utils.reset_time(self)
@defer.inlineCallbacks
def test_fee_and_generous_data(self):
size = 10000000
correct_result = 4.5
daemon = get_test_daemon(generous=True, with_fee=True)
daemon = get_test_daemon(Config(is_generous_host=True), with_fee=True)
result = yield f2d(daemon.get_est_cost("test", size))
self.assertEqual(result, correct_result)
@defer.inlineCallbacks
def test_fee_and_ungenerous_data(self):
conf = Config(is_generous_host=False)
size = 10000000
fake_fee_amount = 4.5
data_rate = conf.ADJUSTABLE_SETTINGS['data_rate'][1]
correct_result = size / 10 ** 6 * data_rate + fake_fee_amount
daemon = get_test_daemon(generous=False, with_fee=True)
correct_result = size / 10 ** 6 * conf.data_rate + fake_fee_amount
daemon = get_test_daemon(conf, with_fee=True)
result = yield f2d(daemon.get_est_cost("test", size))
self.assertEqual(result, round(correct_result, 1))
@@ -105,16 +105,16 @@ class TestCostEst(unittest.TestCase):
def test_generous_data_and_no_fee(self):
size = 10000000
correct_result = 0.0
daemon = get_test_daemon(generous=True)
daemon = get_test_daemon(Config(is_generous_host=True))
result = yield f2d(daemon.get_est_cost("test", size))
self.assertEqual(result, correct_result)
@defer.inlineCallbacks
def test_ungenerous_data_and_no_fee(self):
conf = Config(is_generous_host=False)
size = 10000000
data_rate = conf.ADJUSTABLE_SETTINGS['data_rate'][1]
correct_result = size / 10 ** 6 * data_rate
daemon = get_test_daemon(generous=False)
correct_result = size / 10 ** 6 * conf.data_rate
daemon = get_test_daemon(conf)
result = yield f2d(daemon.get_est_cost("test", size))
self.assertEqual(result, round(correct_result, 1))
@@ -125,10 +125,8 @@ class TestJsonRpc(unittest.TestCase):
def noop():
return None
mock_conf_settings(self)
test_utils.reset_time(self)
self.test_daemon = get_test_daemon()
self.test_daemon.wallet_manager.is_first_run = False
self.test_daemon = get_test_daemon(Config())
self.test_daemon.wallet_manager.get_best_blockhash = noop
@defer.inlineCallbacks
@@ -147,9 +145,8 @@ class TestJsonRpc(unittest.TestCase):
class TestFileListSorting(unittest.TestCase):
def setUp(self):
mock_conf_settings(self)
test_utils.reset_time(self)
self.test_daemon = get_test_daemon()
self.test_daemon = get_test_daemon(Config())
self.test_daemon.file_manager.lbry_files = self._get_fake_lbry_files()
self.test_points_paid = [
+4 -5
View File
@@ -11,14 +11,13 @@ from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.p2p.RateLimiter import DummyRateLimiter
from lbrynet.p2p.client.DownloadManager import DownloadManager
from lbrynet.extras.daemon import Downloader
from lbrynet.extras.daemon import ExchangeRateManager
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
from lbrynet.extras.wallet import LbryWalletManager
from tests.mocks import mock_conf_settings
from lbrynet.conf import Config
class MocDownloader:
@@ -70,7 +69,7 @@ def moc_pay_key_fee(d):
class GetStreamTests(unittest.TestCase):
def init_getstream_with_mocs(self):
mock_conf_settings(self)
conf = Config()
sd_identifier = mock.Mock(spec=StreamDescriptorIdentifier)
wallet = mock.Mock(spec=LbryWalletManager)
@@ -83,7 +82,7 @@ class GetStreamTests(unittest.TestCase):
disable_max_key_fee = False
data_rate = {'currency': "LBC", 'amount': 0, 'address': ''}
getstream = Downloader.GetStream(
sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder, DummyRateLimiter(), prm,
conf, sd_identifier, wallet, exchange_rate_manager, blob_manager, peer_finder, DummyRateLimiter(), prm,
storage, max_key_fee, disable_max_key_fee, timeout=3, data_rate=data_rate
)
getstream.download_manager = mock.Mock(spec=DownloadManager)
@@ -1,8 +1,7 @@
import unittest
from lbrynet.schema.fee import Fee
from lbrynet.extras.daemon import ExchangeRateManager
from lbrynet.extras.daemon import exchange_rate_manager
from lbrynet.p2p.Error import InvalidExchangeRateResponse
from twisted.trial import unittest
from twisted.internet import defer
from tests import test_utils
from tests.mocks import ExchangeRateManager as DummyExchangeRateManager
from tests.mocks import BTCLBCFeed, USDBTCFeed
@@ -36,9 +35,9 @@ class ExchangeRateTest(unittest.TestCase):
def test_invalid_rates(self):
with self.assertRaises(ValueError):
ExchangeRateManager.ExchangeRate('USDBTC', 0, test_utils.DEFAULT_ISO_TIME)
exchange_rate_manager.ExchangeRate('USDBTC', 0, test_utils.DEFAULT_ISO_TIME)
with self.assertRaises(ValueError):
ExchangeRateManager.ExchangeRate('USDBTC', -1, test_utils.DEFAULT_ISO_TIME)
exchange_rate_manager.ExchangeRate('USDBTC', -1, test_utils.DEFAULT_ISO_TIME)
class FeeTest(unittest.TestCase):
@@ -65,7 +64,7 @@ class FeeTest(unittest.TestCase):
def test_missing_feed(self):
# test when a feed is missing for conversion
fee = Fee({
'currency':'USD',
'currency': 'USD',
'amount': 1.0,
'address': "bRcHraa8bYJZL7vkh5sNmGwPDERFUjGPP9"
})
@@ -80,107 +79,96 @@ class FeeTest(unittest.TestCase):
class LBRYioFeedTest(unittest.TestCase):
@defer.inlineCallbacks
def test_handle_response(self):
feed = ExchangeRateManager.LBRYioFeed()
feed = exchange_rate_manager.LBRYioFeed()
response = '{\"data\": {\"fresh\": 0, \"lbc_usd\": 0.05863062523378918, ' \
'\"lbc_btc\": 5.065289549855739e-05, \"btc_usd\": 1157.498}, ' \
'\"success\": true, \"error\": null}'
out = yield feed._handle_response(response)
out = feed._handle_response(response)
expected = 1.0 / 5.065289549855739e-05
self.assertEqual(expected, out)
response = '{}'
with self.assertRaises(InvalidExchangeRateResponse):
out = yield feed._handle_response(response)
out = feed._handle_response(response)
response = '{"success":true,"result":[]}'
with self.assertRaises(InvalidExchangeRateResponse):
out = yield feed._handle_response(response)
out = feed._handle_response(response)
class LBRYioBTCFeedTest(unittest.TestCase):
@defer.inlineCallbacks
def test_handle_response(self):
feed = ExchangeRateManager.LBRYioBTCFeed()
class TestExchangeRateFeeds(unittest.TestCase):
def test_handle_lbryio_btc_response(self):
feed = exchange_rate_manager.LBRYioBTCFeed()
response = '{\"data\": {\"fresh\": 0, \"lbc_usd\": 0.05863062523378918, ' \
'\"lbc_btc\": 5.065289549855739e-05, \"btc_usd\": 1157.498}, ' \
'\"success\": true, \"error\": null}'
out = yield feed._handle_response(response)
out = feed._handle_response(response)
expected = 1.0 / 1157.498
self.assertEqual(expected, out)
response = '{}'
with self.assertRaises(InvalidExchangeRateResponse):
out = yield feed._handle_response(response)
out = feed._handle_response(response)
response = '{"success":true,"result":[]}'
with self.assertRaises(InvalidExchangeRateResponse):
out = yield feed._handle_response(response)
out = feed._handle_response(response)
class CryptonatorFeedTest(unittest.TestCase):
@defer.inlineCallbacks
def test_handle_response(self):
feed = ExchangeRateManager.CryptonatorFeed()
def test_handle_cryptonator_lbc_response(self):
feed = exchange_rate_manager.CryptonatorFeed()
response = '{\"ticker\":{\"base\":\"BTC\",\"target\":\"LBC\",\"price\":\"23657.44026496\"' \
',\"volume\":\"\",\"change\":\"-5.59806916\"},\"timestamp\":1507470422' \
',\"success\":true,\"error\":\"\"}'
out = yield feed._handle_response(response)
out = feed._handle_response(response)
expected = 23657.44026496
self.assertEqual(expected, out)
response = '{}'
with self.assertRaises(InvalidExchangeRateResponse):
out = yield feed._handle_response(response)
out = feed._handle_response(response)
response = '{"success":true,"ticker":{}}'
with self.assertRaises(InvalidExchangeRateResponse):
out = yield feed._handle_response(response)
out = feed._handle_response(response)
class CryptonatorBTCFeedTest(unittest.TestCase):
@defer.inlineCallbacks
def test_handle_response(self):
feed = ExchangeRateManager.CryptonatorBTCFeed()
def test_handle_cryptonator_btc_response(self):
feed = exchange_rate_manager.CryptonatorBTCFeed()
response = '{\"ticker\":{\"base\":\"USD\",\"target\":\"BTC\",\"price\":\"0.00022123\",' \
'\"volume\":\"\",\"change\":\"-0.00000259\"},\"timestamp\":1507471141,' \
'\"success\":true,\"error\":\"\"}'
out = yield feed._handle_response(response)
out = feed._handle_response(response)
expected = 0.00022123
self.assertEqual(expected, out)
response = '{}'
with self.assertRaises(InvalidExchangeRateResponse):
out = yield feed._handle_response(response)
out = feed._handle_response(response)
response = '{"success":true,"ticker":{}}'
with self.assertRaises(InvalidExchangeRateResponse):
out = yield feed._handle_response(response)
out = feed._handle_response(response)
class BittrexFeedTest(unittest.TestCase):
@defer.inlineCallbacks
def test_handle_response(self):
feed = ExchangeRateManager.BittrexFeed()
def test_handle_bittrex_response(self):
feed = exchange_rate_manager.BittrexFeed()
response = '{"success":true,"message":"","result":[{"Id":6902471,"TimeStamp":"2017-02-2'\
'7T23:41:52.213","Quantity":56.12611239,"Price":0.00001621,"Total":0.00090980,"FillType":"'\
'PARTIAL_FILL","OrderType":"SELL"},{"Id":6902403,"TimeStamp":"2017-02-27T23:31:40.463","Qu'\
'antity":430.99988180,"Price":0.00001592,"Total":0.00686151,"FillType":"PARTIAL_FILL","Ord'\
'erType":"SELL"}]}'
out = yield feed._handle_response(response)
out = feed._handle_response(response)
expected = 1.0 / ((0.00090980+0.00686151) / (56.12611239+430.99988180))
self.assertEqual(expected, out)
response = '{}'
with self.assertRaises(InvalidExchangeRateResponse):
out = yield feed._handle_response(response)
out = feed._handle_response(response)
response = '{"success":true,"result":[]}'
with self.assertRaises(InvalidExchangeRateResponse):
out = yield feed._handle_response(response)
out = feed._handle_response(response)
+21 -21
View File
@@ -206,19 +206,19 @@ class TestNIST256pSignatures(UnitTest):
def test_validate_ecdsa_signature(self):
cert = ClaimDict.generate_certificate(nist256p_private_key, curve=NIST256p)
signed = ClaimDict.load_dict(example_010).sign(nist256p_private_key,
claim_address_2, claim_id_1, curve=NIST256p)
claim_address_2, claim_id_1, curve=NIST256p, legacy=True)
self.assertDictEqual(signed.claim_dict, claim_010_signed_nist256p)
signed_copy = ClaimDict.load_protobuf(signed.protobuf)
self.assertEqual(signed_copy.validate_signature(claim_address_2, cert), True)
def test_remove_signature_equals_unsigned(self):
unsigned = ClaimDict.load_dict(example_010)
signed = unsigned.sign(nist256p_private_key, claim_address_1, claim_id_1, curve=NIST256p)
signed = unsigned.sign(nist256p_private_key, claim_address_1, claim_id_1, curve=NIST256p, legacy=True)
self.assertEqual(unsigned.serialized, signed.serialized_no_signature)
def test_fail_to_validate_fake_ecdsa_signature(self):
signed = ClaimDict.load_dict(example_010).sign(nist256p_private_key, claim_address_1,
claim_id_1, curve=NIST256p)
claim_id_1, curve=NIST256p, legacy=True)
signed_copy = ClaimDict.load_protobuf(signed.protobuf)
fake_key = get_signer(NIST256p).generate().private_key.to_pem()
fake_cert = ClaimDict.generate_certificate(fake_key, curve=NIST256p)
@@ -228,7 +228,7 @@ class TestNIST256pSignatures(UnitTest):
def test_fail_to_validate_ecdsa_sig_for_altered_claim(self):
cert = ClaimDict.generate_certificate(nist256p_private_key, curve=NIST256p)
altered = ClaimDict.load_dict(example_010).sign(nist256p_private_key, claim_address_1,
claim_id_1, curve=NIST256p)
claim_id_1, curve=NIST256p, legacy=True)
sd_hash = altered['stream']['source']['source']
altered['stream']['source']['source'] = sd_hash[::-1]
altered_copy = ClaimDict.load_dict(altered.claim_dict)
@@ -245,19 +245,19 @@ class TestNIST384pSignatures(UnitTest):
def test_validate_ecdsa_signature(self):
cert = ClaimDict.generate_certificate(nist384p_private_key, curve=NIST384p)
signed = ClaimDict.load_dict(example_010).sign(nist384p_private_key,
claim_address_2, claim_id_1, curve=NIST384p)
claim_address_2, claim_id_1, curve=NIST384p, legacy=True)
self.assertDictEqual(signed.claim_dict, claim_010_signed_nist384p)
signed_copy = ClaimDict.load_protobuf(signed.protobuf)
self.assertEqual(signed_copy.validate_signature(claim_address_2, cert), True)
def test_remove_signature_equals_unsigned(self):
unsigned = ClaimDict.load_dict(example_010)
signed = unsigned.sign(nist384p_private_key, claim_address_1, claim_id_1, curve=NIST384p)
signed = unsigned.sign(nist384p_private_key, claim_address_1, claim_id_1, curve=NIST384p, legacy=True)
self.assertEqual(unsigned.serialized, signed.serialized_no_signature)
def test_fail_to_validate_fake_ecdsa_signature(self):
signed = ClaimDict.load_dict(example_010).sign(nist384p_private_key, claim_address_1,
claim_id_1, curve=NIST384p)
claim_id_1, curve=NIST384p, legacy=True)
signed_copy = ClaimDict.load_protobuf(signed.protobuf)
fake_key = get_signer(NIST384p).generate().private_key.to_pem()
fake_cert = ClaimDict.generate_certificate(fake_key, curve=NIST384p)
@@ -267,7 +267,7 @@ class TestNIST384pSignatures(UnitTest):
def test_fail_to_validate_ecdsa_sig_for_altered_claim(self):
cert = ClaimDict.generate_certificate(nist384p_private_key, curve=NIST384p)
altered = ClaimDict.load_dict(example_010).sign(nist384p_private_key, claim_address_1,
claim_id_1, curve=NIST384p)
claim_id_1, curve=NIST384p, legacy=True)
sd_hash = altered['stream']['source']['source']
altered['stream']['source']['source'] = sd_hash[::-1]
altered_copy = ClaimDict.load_dict(altered.claim_dict)
@@ -285,7 +285,7 @@ class TestSECP256k1Signatures(UnitTest):
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_2,
claim_id_1, curve=SECP256k1)
claim_id_1, curve=SECP256k1, legacy=True)
self.assertDictEqual(signed.claim_dict, claim_010_signed_secp256k1)
signed_copy = ClaimDict.load_protobuf(signed.protobuf)
self.assertEqual(signed_copy.validate_signature(claim_address_2, cert), True)
@@ -300,19 +300,19 @@ class TestSECP256k1Signatures(UnitTest):
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_2,
claim_id_1, curve=SECP256k1)
claim_id_1, curve=SECP256k1, legacy=True)
self.assertDictEqual(signed.claim_dict, claim_010_signed_secp256k1)
signed_copy = ClaimDict.load_protobuf(signed.protobuf)
self.assertRaises(Exception, signed_copy.validate_signature, None, cert)
def test_remove_signature_equals_unsigned(self):
unsigned = ClaimDict.load_dict(example_010)
signed = unsigned.sign(secp256k1_private_key, claim_address_1, claim_id_1, curve=SECP256k1)
signed = unsigned.sign(secp256k1_private_key, claim_address_1, claim_id_1, curve=SECP256k1, legacy=True)
self.assertEqual(unsigned.serialized, signed.serialized_no_signature)
def test_fail_to_validate_fake_ecdsa_signature(self):
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_1,
claim_id_1, curve=SECP256k1)
claim_id_1, curve=SECP256k1, legacy=True)
signed_copy = ClaimDict.load_protobuf(signed.protobuf)
fake_key = get_signer(SECP256k1).generate().private_key.to_pem()
fake_cert = ClaimDict.generate_certificate(fake_key, curve=SECP256k1)
@@ -322,7 +322,7 @@ class TestSECP256k1Signatures(UnitTest):
def test_fail_to_validate_ecdsa_sig_for_altered_claim(self):
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
altered = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_1,
claim_id_1, curve=SECP256k1)
claim_id_1, curve=SECP256k1, legacy=True)
sd_hash = altered['stream']['source']['source']
altered['stream']['source']['source'] = sd_hash[::-1]
altered_copy = ClaimDict.load_dict(altered.claim_dict)
@@ -335,7 +335,7 @@ class TestDetachedNamedSECP256k1Signatures(UnitTest):
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_2, claim_id_1,
curve=SECP256k1, name='example', force_detached=True)
curve=SECP256k1, name='example')
signed_copy = ClaimDict.deserialize(signed.serialized)
self.assertEqual(signed_copy.validate_signature(claim_address_2, cert, name='example'), True)
@@ -343,7 +343,7 @@ class TestDetachedNamedSECP256k1Signatures(UnitTest):
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_2, claim_id_1,
curve=SECP256k1, name='example', force_detached=True)
curve=SECP256k1, name='example')
self.assertEqual(
signed.claim_dict['publisherSignature']['detached_signature'],
binascii.hexlify(signed.serialized).decode()
@@ -408,13 +408,13 @@ class TestDetachedNamedSECP256k1Signatures(UnitTest):
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
self.assertRaises(Exception, ClaimDict.load_dict(example_010).sign, secp256k1_private_key,
None, claim_id_1, curve=SECP256k1, name='example', force_detached=True)
None, claim_id_1, curve=SECP256k1, name='example')
def test_fail_to_validate_with_no_claim_address(self):
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_2, claim_id_1,
curve=SECP256k1, name='example', force_detached=True)
curve=SECP256k1, name='example')
signed_copy = ClaimDict.load_protobuf(signed.protobuf)
self.assertRaises(Exception, signed_copy.validate_signature, None, cert, name='example')
@@ -422,19 +422,19 @@ class TestDetachedNamedSECP256k1Signatures(UnitTest):
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
self.assertDictEqual(cert.claim_dict, secp256k1_cert)
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_2, claim_id_1,
curve=SECP256k1, name='example', force_detached=True)
curve=SECP256k1, name='example')
signed_copy = ClaimDict.load_protobuf(signed.protobuf)
self.assertRaises(Exception, signed_copy.validate_signature, None, cert, name=None)
def test_remove_signature_equals_unsigned(self):
unsigned = ClaimDict.load_dict(example_010)
signed = unsigned.sign(secp256k1_private_key, claim_address_1, claim_id_1,
curve=SECP256k1, name='example', force_detached=True)
curve=SECP256k1, name='example')
self.assertEqual(unsigned.serialized, signed.serialized_no_signature)
def test_fail_to_validate_fake_ecdsa_signature(self):
signed = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_1, claim_id_1,
curve=SECP256k1, name='example', force_detached=True)
curve=SECP256k1, name='example')
signed_copy = ClaimDict.deserialize(signed.serialized)
fake_key = get_signer(SECP256k1).generate().private_key.to_pem()
fake_cert = ClaimDict.generate_certificate(fake_key, curve=SECP256k1)
@@ -444,7 +444,7 @@ class TestDetachedNamedSECP256k1Signatures(UnitTest):
def test_fail_to_validate_ecdsa_sig_for_altered_claim(self):
cert = ClaimDict.generate_certificate(secp256k1_private_key, curve=SECP256k1)
altered = ClaimDict.load_dict(example_010).sign(secp256k1_private_key, claim_address_1, claim_id_1,
curve=SECP256k1, name='example', force_detached=True)
curve=SECP256k1, name='example')
original_serialization = altered.serialized
sd_hash = altered['stream']['source']['source']
altered['stream']['source']['source'] = sd_hash[::-1]
+8 -9
View File
@@ -44,8 +44,7 @@ class CLITest(unittest.TestCase):
with contextlib.redirect_stdout(actual_output):
main(['help'])
actual_output = actual_output.getvalue()
self.assertSubstring('lbrynet - LBRY command line client.', actual_output)
self.assertSubstring('USAGE', actual_output)
self.assertSubstring('usage: lbrynet [--version] [-h]', actual_output)
def test_help_for_command_command(self):
actual_output = StringIO()
@@ -64,7 +63,7 @@ class CLITest(unittest.TestCase):
def test_version_command(self):
actual_output = StringIO()
with contextlib.redirect_stdout(actual_output):
main(['version'])
main(['--version'])
self.assertEqual(
actual_output.getvalue().strip(),
"lbrynet {lbrynet_version}".format(**get_platform())
@@ -72,12 +71,12 @@ class CLITest(unittest.TestCase):
def test_invalid_command(self):
actual_output = StringIO()
with contextlib.redirect_stdout(actual_output):
main(['publish1'])
self.assertEqual(
actual_output.getvalue().strip(),
"publish1 is not a valid command."
)
with contextlib.redirect_stderr(actual_output):
try:
main(['publish1'])
except SystemExit:
pass
self.assertSubstring("invalid choice: 'publish1'", actual_output.getvalue())
def test_valid_command_daemon_not_started(self):
actual_output = StringIO()
+147 -99
View File
@@ -1,121 +1,169 @@
import os
import json
import sys
import types
import tempfile
import shutil
from unittest import skipIf
from twisted.trial import unittest
from twisted.internet import defer
from lbrynet import conf
import unittest
import argparse
from lbrynet.conf import Config, BaseConfig, String, Integer, Toggle, Servers, NOT_SET
from lbrynet.p2p.Error import InvalidCurrencyError
class SettingsTest(unittest.TestCase):
def setUp(self):
os.environ['LBRY_TEST'] = 'test_string'
class TestConfig(BaseConfig):
test_str = String('str help', 'the default', previous_names=['old_str'])
test_int = Integer('int help', 9)
test_toggle = Toggle('toggle help', False)
servers = Servers('servers help', [('localhost', 80)])
def tearDown(self):
del os.environ['LBRY_TEST']
def get_mock_config_instance(self):
settings = {'test': (str, '')}
env = conf.Env(**settings)
self.tmp_dir = tempfile.mkdtemp()
self.addCleanup(lambda : defer.succeed(shutil.rmtree(self.tmp_dir)))
return conf.Config({}, settings, environment=env, data_dir=self.tmp_dir, wallet_dir=self.tmp_dir, download_dir=self.tmp_dir)
class ConfigurationTests(unittest.TestCase):
def test_envvar_is_read(self):
settings = self.get_mock_config_instance()
self.assertEqual('test_string', settings['test'])
@unittest.skipIf('linux' not in sys.platform, 'skipping linux only test')
def test_linux_defaults(self):
c = Config()
self.assertEqual(c.data_dir, os.path.expanduser('~/.local/share/lbry/lbrynet'))
self.assertEqual(c.wallet_dir, os.path.expanduser('~/.local/share/lbry/lbryum'))
self.assertEqual(c.download_dir, os.path.expanduser('~/Downloads'))
self.assertEqual(c.config, os.path.expanduser('~/.local/share/lbry/lbrynet/daemon_settings.yml'))
self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi')
self.assertEqual(c.log_file_path, os.path.expanduser('~/.local/share/lbry/lbrynet/lbrynet.log'))
def test_setting_can_be_overridden(self):
settings = self.get_mock_config_instance()
settings['test'] = 'my_override'
self.assertEqual('my_override', settings['test'])
def test_search_order(self):
c = TestConfig()
c.runtime = {'test_str': 'runtime'}
c.arguments = {'test_str': 'arguments'}
c.environment = {'test_str': 'environment'}
c.persisted = {'test_str': 'persisted'}
self.assertEqual(c.test_str, 'runtime')
c.runtime = {}
self.assertEqual(c.test_str, 'arguments')
c.arguments = {}
self.assertEqual(c.test_str, 'environment')
c.environment = {}
self.assertEqual(c.test_str, 'persisted')
c.persisted = {}
self.assertEqual(c.test_str, 'the default')
def test_setting_can_be_updated(self):
settings = self.get_mock_config_instance()
settings.update({'test': 'my_update'})
self.assertEqual('my_update', settings['test'])
def test_arguments(self):
parser = argparse.ArgumentParser()
parser.add_argument("--test-str")
args = parser.parse_args(['--test-str', 'blah'])
c = TestConfig.create_from_arguments(args)
self.assertEqual(c.test_str, 'blah')
c.arguments = {}
self.assertEqual(c.test_str, 'the default')
def test_setting_is_in_dict(self):
settings = self.get_mock_config_instance()
setting_dict = settings.get_current_settings_dict()
self.assertEqual({'test': 'test_string'}, setting_dict)
def test_environment(self):
c = TestConfig()
self.assertEqual(c.test_str, 'the default')
c.set_environment({'LBRY_TEST_STR': 'from environ'})
self.assertEqual(c.test_str, 'from environ')
def test_invalid_setting_raises_exception(self):
settings = self.get_mock_config_instance()
self.assertRaises(KeyError, settings.set, 'invalid_name', 123)
def test_persisted(self):
with tempfile.TemporaryDirectory() as temp_dir:
def test_invalid_data_type_raises_exception(self):
settings = self.get_mock_config_instance()
self.assertIsNone(settings.set('test', 123))
self.assertRaises(KeyError, settings.set, 'test', 123, ('fake_data_type',))
c = TestConfig.create_from_arguments(
types.SimpleNamespace(config=os.path.join(temp_dir, 'settings.yml'))
)
def test_setting_precedence(self):
settings = self.get_mock_config_instance()
settings.set('test', 'cli_test_string', data_types=(conf.TYPE_CLI,))
self.assertEqual('cli_test_string', settings['test'])
settings.set('test', 'this_should_not_take_precedence', data_types=(conf.TYPE_ENV,))
self.assertEqual('cli_test_string', settings['test'])
settings.set('test', 'runtime_takes_precedence', data_types=(conf.TYPE_RUNTIME,))
self.assertEqual('runtime_takes_precedence', settings['test'])
# settings.yml doesn't exist on file system
self.assertFalse(c.persisted.exists)
self.assertEqual(c.test_str, 'the default')
def test_max_key_fee_set(self):
fixed_default = {'CURRENCIES':{'BTC':{'type':'crypto'}}}
adjustable_settings = {'max_key_fee': (json.loads, {'currency':'USD', 'amount':1})}
env = conf.Env(**adjustable_settings)
settings = conf.Config(fixed_default, adjustable_settings, environment=env)
self.assertEqual(c.modify_order, [c.runtime])
with c.update_config():
self.assertEqual(c.modify_order, [c.runtime, c.persisted])
c.test_str = 'original'
self.assertEqual(c.modify_order, [c.runtime])
with self.assertRaises(InvalidCurrencyError):
settings.set('max_key_fee', {'currency':'USD', 'amount':1})
# share_usage_data has been saved to settings file
self.assertTrue(c.persisted.exists)
with open(c.config, 'r') as fd:
self.assertEqual(fd.read(), 'test_str: original\n')
valid_setting = {'currency':'BTC', 'amount':1}
settings.set('max_key_fee', valid_setting)
out = settings.get('max_key_fee')
self.assertEqual(out, valid_setting)
# load the settings file and check share_usage_data is false
c = TestConfig.create_from_arguments(
types.SimpleNamespace(config=os.path.join(temp_dir, 'settings.yml'))
)
self.assertTrue(c.persisted.exists)
self.assertEqual(c.test_str, 'original')
def test_data_dir(self):
# check if these directories are returned as string and not unicode
# otherwise there will be problems when calling os.path.join on
# unicode directory names with string file names
settings = conf.Config({}, {})
self.assertEqual(str, type(settings.download_dir))
self.assertEqual(str, type(settings.data_dir))
self.assertEqual(str, type(settings.wallet_dir))
# setting in runtime overrides config
self.assertNotIn('test_str', c.runtime)
c.test_str = 'from runtime'
self.assertIn('test_str', c.runtime)
self.assertEqual(c.test_str, 'from runtime')
@skipIf('win' in sys.platform, 'fix me!')
def test_load_save_config_file(self):
# setup settings
adjustable_settings = {'lbryum_servers': (list, [])}
env = conf.Env(**adjustable_settings)
settings = conf.Config({}, adjustable_settings, environment=env)
conf.settings = settings
# setup tempfile
conf_entry = b"lbryum_servers: ['localhost:50001', 'localhost:50002']\n"
with tempfile.NamedTemporaryFile(suffix='.yml') as conf_file:
conf_file.write(conf_entry)
conf_file.seek(0)
conf.conf_file = conf_file.name
# load and save settings from conf file
settings.load_conf_file_settings()
settings.save_conf_file_settings()
# test if overwritten entry equals original entry
# use decoded versions, because format might change without
# changing the interpretation
decoder = conf.settings_decoders['.yml']
conf_decoded = decoder(conf_entry)
conf_entry_new = conf_file.read()
conf_decoded_new = decoder(conf_entry_new)
self.assertEqual(conf_decoded, conf_decoded_new)
# without context manager NOT_SET only clears it in runtime location
c.test_str = NOT_SET
self.assertNotIn('test_str', c.runtime)
self.assertEqual(c.test_str, 'original')
def test_load_file(self):
settings = self.get_mock_config_instance()
# clear it in persisted as well by using context manager
self.assertIn('test_str', c.persisted)
with c.update_config():
c.test_str = NOT_SET
self.assertNotIn('test_str', c.persisted)
self.assertEqual(c.test_str, 'the default')
with open(c.config, 'r') as fd:
self.assertEqual(fd.read(), '{}\n')
# invalid extensions
for filename in ('monkey.yymmll', 'monkey'):
settings.file_name = filename
with open(os.path.join(self.tmp_dir, filename), "w"):
pass
with self.assertRaises(ValueError):
settings.load_conf_file_settings()
def test_persisted_upgrade(self):
with tempfile.TemporaryDirectory() as temp_dir:
config = os.path.join(temp_dir, 'settings.yml')
with open(config, 'w') as fd:
fd.write('old_str: old stuff\n')
c = TestConfig.create_from_arguments(
types.SimpleNamespace(config=config)
)
self.assertEqual(c.test_str, 'old stuff')
self.assertNotIn('old_str', c.persisted)
with open(config, 'w') as fd:
fd.write('test_str: old stuff\n')
def test_validation(self):
c = TestConfig()
with self.assertRaisesRegex(AssertionError, 'must be a string'):
c.test_str = 9
with self.assertRaisesRegex(AssertionError, 'must be an integer'):
c.test_int = 'hi'
with self.assertRaisesRegex(AssertionError, 'must be a true/false'):
c.test_toggle = 'hi'
def test_file_extension_validation(self):
with self.assertRaisesRegex(AssertionError, "'.json' is not supported"):
TestConfig.create_from_arguments(
types.SimpleNamespace(config=os.path.join('settings.json'))
)
def test_serialize_deserialize(self):
with tempfile.TemporaryDirectory() as temp_dir:
c = TestConfig.create_from_arguments(
types.SimpleNamespace(config=os.path.join(temp_dir, 'settings.yml'))
)
self.assertEqual(c.servers, [('localhost', 80)])
with c.update_config():
c.servers = [('localhost', 8080)]
with open(c.config, 'r+') as fd:
self.assertEqual(fd.read(), 'servers:\n- localhost:8080\n')
fd.write('servers:\n - localhost:5566\n')
c = TestConfig.create_from_arguments(
types.SimpleNamespace(config=os.path.join(temp_dir, 'settings.yml'))
)
self.assertEqual(c.servers, [('localhost', 5566)])
def test_max_key_fee(self):
with tempfile.TemporaryDirectory() as temp_dir:
config = os.path.join(temp_dir, 'settings.yml')
with open(config, 'w') as fd:
fd.write('max_key_fee: \'{"currency":"USD", "amount":1}\'\n')
c = Config.create_from_arguments(
types.SimpleNamespace(config=config)
)
self.assertEqual(c.max_key_fee['currency'], 'USD')
self.assertEqual(c.max_key_fee['amount'], 1)
with self.assertRaises(InvalidCurrencyError):
c.max_key_fee = {'currency': 'BCH', 'amount': 1}
with c.update_config():
c.max_key_fee = {'currency': 'BTC', 'amount': 1}
with open(config, 'r') as fd:
self.assertEqual(fd.read(), 'max_key_fee: \'{"currency": "BTC", "amount": 1}\'\n')
-53
View File
@@ -1,53 +0,0 @@
from io import StringIO
import logging
from unittest import mock
from unittest import skipIf
from twisted.internet import defer
from twisted.trial import unittest
from lbrynet import custom_logger
from tests.test_utils import is_android
class TestLogger(unittest.TestCase):
def raiseError(self):
raise Exception('terrible things happened')
def triggerErrback(self, callback=None):
d = defer.Deferred()
d.addCallback(lambda _: self.raiseError())
d.addErrback(self.log.fail(callback), 'My message')
d.callback(None)
return d
def setUp(self):
self.log = custom_logger.Logger('test')
self.stream = StringIO()
handler = logging.StreamHandler(self.stream)
handler.setFormatter(logging.Formatter("%(filename)s:%(lineno)d - %(message)s"))
self.log.addHandler(handler)
@skipIf(is_android(),
'Test cannot pass on Android because the tests package is compiled '
'which results in a different method call stack')
def test_can_log_failure(self):
def output_lines():
return self.stream.getvalue().split('\n')
# the line number could change if this file gets refactored
expected_first_line = 'test_customLogger.py:20 - My message: terrible things happened'
# testing the entirety of the message is futile as the
# traceback will depend on the system the test is being run on
# but hopefully these two tests are good enough
d = self.triggerErrback()
d.addCallback(lambda _: self.assertEquals(expected_first_line, output_lines()[0]))
d.addCallback(lambda _: self.assertEqual(10, len(output_lines())))
return d
def test_can_log_failure_with_callback(self):
callback = mock.Mock()
d = self.triggerErrback(callback)
d.addCallback(lambda _: callback.assert_called_once_with(mock.ANY))
return d