Compare commits

..
Author SHA1 Message Date
Brannon King a58ebcbc53 triggers never activated 2020-02-05 13:18:15 -07:00
55 changed files with 352 additions and 776 deletions

No files matched your search

-79
View File
@@ -1,79 +0,0 @@
name: ci
on: pull_request
jobs:
lint:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- run: make install tools
- run: make lint
tests-unit:
name: "tests / unit"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- run: make install tools
- working-directory: lbry
env:
HOME: /tmp
run: coverage run -p --source=lbry -m unittest discover -vv tests.unit
tests-integration:
name: "tests / integration"
runs-on: ubuntu-latest
strategy:
matrix:
test:
- datanetwork
- blockchain
- other
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- if: matrix.test == 'other'
run: sudo apt install -y --no-install-recommends ffmpeg
- run: pip install tox-travis
- run: tox -e ${{ matrix.test }}
build:
needs: ["lint", "tests-unit", "tests-integration"]
name: "build"
strategy:
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- name: Setup
run: |
pip install pyinstaller
pip install -e .
- if: startsWith(matrix.os, 'windows') == false
name: Build & Run (Unix)
run: |
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
chmod +x dist/lbrynet
dist/lbrynet --version
- if: startsWith(matrix.os, 'windows')
name: Build & Run (Windows)
run: |
pip install pywin32
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
dist/lbrynet.exe --version
+4 -5
View File
@@ -46,14 +46,14 @@ test:datanetwork-integration:
test:blockchain-integration:
stage: test
script:
- apt-get update
- apt-get install -y --no-install-recommends ffmpeg
- pip install tox-travis
- tox -e blockchain
test:other-integration:
stage: test
script:
- apt-get update
- apt-get install -y --no-install-recommends ffmpeg
- pip install tox-travis
- tox -e other
@@ -84,15 +84,14 @@ test:json-api:
build:linux:
extends: .build
image: ubuntu:16.04
image: ubuntu:18.04 # cant use python3.7 image - binary won't run on ubuntu
variables:
OS: linux
before_script:
- apt-get update
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
- add-apt-repository -y ppa:deadsnakes/ppa
- apt-get update
- apt-get install -y --no-install-recommends python3.7-dev
- apt-get install -y --no-install-recommends python3.7-dev python3-setuptools python3-wheel
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
build:mac:
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.60.1"
__version__ = "0.58.1"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+10 -16
View File
@@ -558,7 +558,6 @@ class Config(CLIConfig):
"Don't download streams with fees exceeding this amount. When set to "
"null, the amount is unbounded.", {'currency': 'USD', 'amount': 50.0}
)
max_wallet_server_fee = String("Maximum daily LBC amount allowed as payment for wallet servers.", "0.0")
# reflector settings
reflect_streams = Toggle(
@@ -574,15 +573,15 @@ class Config(CLIConfig):
('reflector.lbry.com', 5566)
])
lbryum_servers = Servers("SPV wallet servers", [
('spv11.lbry.com', 50001),
('spv12.lbry.com', 50001),
('spv13.lbry.com', 50001),
('spv14.lbry.com', 50001),
('spv15.lbry.com', 50001),
('spv16.lbry.com', 50001),
('spv17.lbry.com', 50001),
('spv18.lbry.com', 50001),
('spv19.lbry.com', 50001),
('spv1.lbry.com', 50001),
('spv2.lbry.com', 50001),
('spv3.lbry.com', 50001),
('spv4.lbry.com', 50001),
('spv5.lbry.com', 50001),
('spv6.lbry.com', 50001),
('spv7.lbry.com', 50001),
('spv8.lbry.com', 50001),
('spv9.lbry.com', 50001),
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.com', 4444), # US EAST
@@ -602,7 +601,7 @@ class Config(CLIConfig):
save_files = Toggle("Save downloaded files when calling `get` by default", True)
components_to_skip = Strings("components which will be skipped during start-up of daemon", [])
share_usage_data = Toggle(
"Whether to share usage stats and diagnostic info with LBRY.", False,
"Whether to share usage stats and diagnostic info with LBRY.", True,
previous_names=['upload_log', 'upload_log', 'share_debug_info']
)
track_bandwidth = Toggle("Track bandwidth usage", True)
@@ -617,11 +616,6 @@ class Config(CLIConfig):
"Strategy to use when selecting UTXOs for a transaction",
STRATEGIES, "standard")
save_resolved_claims = Toggle(
"Save content claims to the database when they are resolved to keep file_list up to date, "
"only disable this if file_x commands are not needed", True
)
@property
def streaming_host(self):
return self.streaming_server.split(':')[0]
+1 -4
View File
@@ -53,13 +53,10 @@ Code | Name | Message
407 | DataDownload | Failed to download blob. *generic*
410 | Resolve | Failed to resolve '{url}'.
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
420 | KeyFeeAboveMaxAllowed | {message}
421 | InvalidPassword | Password is invalid.
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
431 | ServerPaymentInvalidAddress | Invalid address from wallet server: '{address}' - skipping payment round.
432 | ServerPaymentWalletLocked | Cannot spend funds with locked wallet, skipping payment round.
433 | ServerPaymentFeeAboveMaxAllowed | Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.
**5xx** | Blob | **Blobs**
500 | BlobNotFound | Blob not found.
501 | BlobPermissionDenied | Permission denied to read blob.
+3 -56
View File
@@ -1,4 +1,4 @@
from .base import BaseError, claim_id
from .base import BaseError
class UserInputError(BaseError):
@@ -16,22 +16,18 @@ class CommandError(UserInputError):
class CommandDoesNotExistError(CommandError):
def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' does not exist.")
class CommandDeprecatedError(CommandError):
def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' is deprecated.")
class CommandInvalidArgumentError(CommandError):
def __init__(self, argument, command):
self.argument = argument
self.command = command
super().__init__(f"Invalid argument '{argument}' to command '{command}'.")
@@ -41,7 +37,6 @@ class CommandTemporarilyUnavailableError(CommandError):
"""
def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' is temporarily unavailable.")
@@ -51,7 +46,6 @@ class CommandPermanentlyUnavailableError(CommandError):
"""
def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' is permanently unavailable.")
@@ -64,15 +58,12 @@ class InputValueError(UserInputError, ValueError):
class GenericInputValueError(InputValueError):
def __init__(self, value, argument):
self.value = value
self.argument = argument
super().__init__(f"The value '{value}' for argument '{argument}' is not valid.")
class InputValueIsNoneError(InputValueError):
def __init__(self, argument):
self.argument = argument
super().__init__(f"None or null is not valid value for argument '{argument}'.")
@@ -88,7 +79,6 @@ class ConfigWriteError(ConfigurationError):
"""
def __init__(self, path):
self.path = path
super().__init__(f"Cannot write configuration file '{path}'.")
@@ -98,7 +88,6 @@ class ConfigReadError(ConfigurationError):
"""
def __init__(self, path):
self.path = path
super().__init__(f"Cannot find provided configuration file '{path}'.")
@@ -108,21 +97,18 @@ class ConfigParseError(ConfigurationError):
"""
def __init__(self, path):
self.path = path
super().__init__(f"Failed to parse the configuration file '{path}'.")
class ConfigMissingError(ConfigurationError):
def __init__(self, path):
self.path = path
super().__init__(f"Configuration file '{path}' is missing setting that has no default / fallback.")
class ConfigInvalidError(ConfigurationError):
def __init__(self, path):
self.path = path
super().__init__(f"Configuration file '{path}' has setting with invalid value.")
@@ -202,29 +188,24 @@ class DataDownloadError(WalletError):
class ResolveError(WalletError):
def __init__(self, url):
self.url = url
super().__init__(f"Failed to resolve '{url}'.")
class ResolveTimeoutError(WalletError):
def __init__(self, url):
self.url = url
super().__init__(f"Failed to resolve '{url}' within the timeout.")
class ResolveCensoredError(WalletError):
def __init__(self, url, censor_hash):
self.url = url
self.censor_hash = censor_hash
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.")
def __init__(self, url, censor_id):
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.")
class KeyFeeAboveMaxAllowedError(WalletError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
@@ -237,32 +218,9 @@ class InvalidPasswordError(WalletError):
class IncompatibleWalletServerError(WalletError):
def __init__(self, server, port):
self.server = server
self.port = port
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
class ServerPaymentInvalidAddressError(WalletError):
def __init__(self, address):
self.address = address
super().__init__(f"Invalid address from wallet server: '{address}' - skipping payment round.")
class ServerPaymentWalletLockedError(WalletError):
def __init__(self):
super().__init__("Cannot spend funds with locked wallet, skipping payment round.")
class ServerPaymentFeeAboveMaxAllowedError(WalletError):
def __init__(self, daily_fee, max_fee):
self.daily_fee = daily_fee
self.max_fee = max_fee
super().__init__(f"Daily server fee of {daily_fee} exceeds maximum configured of {max_fee} LBC.")
class BlobError(BaseError):
"""
**Blobs**
@@ -320,35 +278,30 @@ class DownloadCancelledError(BlobError):
class DownloadSDTimeoutError(BlobError):
def __init__(self, download):
self.download = download
super().__init__(f"Failed to download sd blob {download} within timeout.")
class DownloadDataTimeoutError(BlobError):
def __init__(self, download):
self.download = download
super().__init__(f"Failed to download data blobs for sd hash {download} within timeout.")
class InvalidStreamDescriptorError(BlobError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
class InvalidDataError(BlobError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
class InvalidBlobHashError(BlobError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
@@ -361,14 +314,12 @@ class ComponentError(BaseError):
class ComponentStartConditionNotMetError(ComponentError):
def __init__(self, components):
self.components = components
super().__init__(f"Unresolved dependencies for: {components}")
class ComponentsNotStartedError(ComponentError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
@@ -381,20 +332,16 @@ class CurrencyExchangeError(BaseError):
class InvalidExchangeRateResponseError(CurrencyExchangeError):
def __init__(self, source, reason):
self.source = source
self.reason = reason
super().__init__(f"Failed to get exchange rate from {source}: {reason}")
class CurrencyConversionError(CurrencyExchangeError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
class InvalidCurrencyError(CurrencyExchangeError):
def __init__(self, currency):
self.currency = currency
super().__init__(f"Invalid currency: {currency} is not a supported currency.")
-7
View File
@@ -1,9 +1,2 @@
from binascii import hexlify
def claim_id(claim_hash):
return hexlify(claim_hash[::-1]).decode()
class BaseError(Exception):
pass
+4 -17
View File
@@ -13,12 +13,10 @@ class {name}({parents}):{doc}
"""
INIT = """
def __init__({args}):{fields}
def __init__({args}):
super().__init__({format}"{message}")
"""
FUNCTIONS = ['claim_id']
class ErrorClass:
@@ -52,20 +50,10 @@ class ErrorClass:
def get_arguments(self):
args = ['self']
for arg in re.findall('{([a-z0-1_()]+)}', self.message):
for func in FUNCTIONS:
if arg.startswith(f'{func}('):
arg = arg[len(f'{func}('):-1]
break
for arg in re.findall('{([a-z0-1_]+)}', self.message):
args.append(arg)
return args
@staticmethod
def get_fields(args):
if len(args) > 1:
return f''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
return ''
@staticmethod
def get_doc_string(doc):
if doc:
@@ -81,8 +69,7 @@ class ErrorClass:
args = self.get_arguments()
if self.is_leaf:
out.write((CLASS + INIT).format(
name=self.class_name, parents=', '.join(parents),
args=', '.join(args), fields=self.get_fields(args),
name=self.class_name, parents=', '.join(parents), args=', '.join(args),
message=self.message, doc=self.get_doc_string(self.comment), format='f' if len(args) > 1 else ''
))
else:
@@ -115,7 +102,7 @@ def find_parent(stack, child):
def generate(out):
out.write(f"from .base import BaseError, {', '.join(FUNCTIONS)}\n")
out.write('from .base import BaseError\n')
stack = {}
for error in get_errors():
error.render(out, find_parent(stack, error))
+4 -3
View File
@@ -256,9 +256,10 @@ def setup_logging(logger: logging.Logger, args: argparse.Namespace, conf: Config
else:
logger.getChild('lbry').setLevel(logging.DEBUG)
loggly_handler = get_loggly_handler(conf)
loggly_handler.setLevel(logging.ERROR)
logger.getChild('lbry').addHandler(loggly_handler)
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
logger.getChild('lbry').addHandler(loggly_handler)
def run_daemon(args: argparse.Namespace, conf: Config):
+6 -9
View File
@@ -110,6 +110,7 @@ class AnalyticsManager:
self.cookies = {}
self.url = ANALYTICS_ENDPOINT
self._write_key = utils.deobfuscate(ANALYTICS_TOKEN)
self._enabled = conf.share_usage_data
self._tracked_data = collections.defaultdict(list)
self.context = _make_context(system_info.get_platform())
self.installation_id = installation_id
@@ -117,24 +118,20 @@ class AnalyticsManager:
self.task: typing.Optional[asyncio.Task] = None
self.external_ip: typing.Optional[str] = None
@property
def enabled(self):
return self.conf.share_usage_data
@property
def is_started(self):
return self.task is not None
async def start(self):
if self.task is None:
if self._enabled and self.task is None:
self.external_ip = await utils.get_external_ip()
self.task = asyncio.create_task(self.run())
async def run(self):
while True:
if self.enabled:
self.external_ip = await utils.get_external_ip()
await self._send_heartbeat()
await self._send_heartbeat()
await asyncio.sleep(1800)
self.external_ip = await utils.get_external_ip()
def stop(self):
if self.task is not None and not self.task.done():
@@ -157,7 +154,7 @@ class AnalyticsManager:
async def track(self, event: typing.Dict):
"""Send a single tracking event"""
if self.enabled:
if self._enabled:
log.debug('Sending track event: %s', event)
await self._post(event)
-30
View File
@@ -21,7 +21,6 @@ from lbry.extras.daemon.component import Component
from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbry.extras.daemon.storage import SQLiteStorage
from lbry.wallet import WalletManager
from lbry.wallet.usage_payment import WalletServerPayer
log = logging.getLogger(__name__)
@@ -30,7 +29,6 @@ log = logging.getLogger(__name__)
DATABASE_COMPONENT = "database"
BLOB_COMPONENT = "blob_manager"
WALLET_COMPONENT = "wallet"
WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments"
DHT_COMPONENT = "dht"
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
STREAM_MANAGER_COMPONENT = "stream_manager"
@@ -162,34 +160,6 @@ class WalletComponent(Component):
self.wallet_manager = None
class WalletServerPaymentsComponent(Component):
component_name = WALLET_SERVER_PAYMENTS_COMPONENT
depends_on = [WALLET_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.usage_payment_service = WalletServerPayer(
max_fee=self.conf.max_wallet_server_fee, analytics_manager=self.component_manager.analytics_manager,
)
@property
def component(self) -> typing.Optional[WalletServerPayer]:
return self.usage_payment_service
async def start(self):
wallet_manager = self.component_manager.get_component(WALLET_COMPONENT)
await self.usage_payment_service.start(wallet_manager.ledger, wallet_manager.default_wallet)
async def stop(self):
await self.usage_payment_service.stop()
async def get_status(self):
return {
'max_fee': self.usage_payment_service.max_fee,
'running': self.usage_payment_service.running
}
class BlobComponent(Component):
component_name = BLOB_COMPONENT
depends_on = [DATABASE_COMPONENT]
+8 -27
View File
@@ -297,7 +297,6 @@ class Daemon(metaclass=JSONRPCServerType):
def __init__(self, conf: Config, component_manager: typing.Optional[ComponentManager] = None):
self.conf = conf
self.platform_info = system_info.get_platform()
self._video_file_analyzer = VideoFileAnalyzer(conf)
self._node_id = None
self._installation_id = None
@@ -308,7 +307,7 @@ class Daemon(metaclass=JSONRPCServerType):
skip_components=conf.components_to_skip or []
)
self.component_startup_task = None
self._connection_status: typing.Tuple[float, bool] = (self.component_manager.loop.time(), False)
self._connection_status: typing.Tuple[float, bool] = [self.component_manager.loop.time(), False]
logging.getLogger('aiohttp.access').setLevel(logging.WARN)
rpc_app = web.Application()
@@ -358,7 +357,7 @@ class Daemon(metaclass=JSONRPCServerType):
@classmethod
def get_api_definitions(cls):
prefix = 'jsonrpc_'
not_grouped = ['routing_table_get', 'ffmpeg_find']
not_grouped = ['routing_table_get']
api = {
'groups': {
group_name[:-len('_DOC')].lower(): getattr(cls, group_name).strip()
@@ -450,7 +449,7 @@ class Daemon(metaclass=JSONRPCServerType):
async def start(self):
log.info("Starting LBRYNet Daemon")
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
log.info("Platform: %s", json.dumps(self.platform_info, indent=2))
log.info("Platform: %s", json.dumps(system_info.get_platform(), indent=2))
await self.analytics_manager.send_server_startup()
await self.rpc_runner.setup()
await self.streaming_runner.setup()
@@ -765,26 +764,6 @@ class Daemon(metaclass=JSONRPCServerType):
asyncio.get_event_loop().call_later(0, shutdown)
return "Shutting down"
async def jsonrpc_ffmpeg_find(self):
"""
Get ffmpeg installation information
Usage:
ffmpeg_find
Options:
None
Returns:
(dict) Dictionary of ffmpeg information
{
'available': (bool) found ffmpeg,
'which': (str) path to ffmpeg,
'analyze_audio_volume': (bool) should ffmpeg analyze audio
}
"""
return await self._video_file_analyzer.status(recheck=True)
async def jsonrpc_status(self):
"""
Get daemon status
@@ -915,7 +894,9 @@ class Daemon(metaclass=JSONRPCServerType):
'build': (str) "dev" | "qa" | "rc" | "release",
}
"""
return self.platform_info
platform_info = system_info.get_platform()
log.info("Get version info: %s", json.dumps(platform_info))
return platform_info
@requires(WALLET_COMPONENT)
async def jsonrpc_resolve(self, urls: typing.Union[str, list], wallet_id=None):
@@ -2173,7 +2154,7 @@ class Daemon(metaclass=JSONRPCServerType):
List my stream and channel claims.
Usage:
claim_list [--claim_type=<claim_type>...]
claim_list [--claim_type=<claim_type>]
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--page=<page>] [--page_size=<page_size>]
[--resolve]
@@ -4872,7 +4853,7 @@ class Daemon(metaclass=JSONRPCServerType):
async def resolve(self, accounts, urls):
results = await self.ledger.resolve(accounts, urls)
if self.conf.save_resolved_claims and results:
if results:
try:
claims = self.stream_manager._convert_to_old_resolve_output(self.wallet_manager, results)
await self.storage.save_claims_for_resolve([
+1 -2
View File
@@ -294,8 +294,7 @@ class JSONResponseEncoder(JSONEncoder):
'added_on': managed_stream.added_on,
'height': tx_height,
'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height,
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None,
'is_fully_reflected': managed_stream.is_fully_reflected
'timestamp': self.ledger.headers[tx_height]['timestamp'] if 0 < tx_height <= best_height else None
}
def encode_claim(self, claim):
+9 -15
View File
@@ -3,12 +3,10 @@ import json
import logging.handlers
import traceback
import typing
from aiohttp.client_exceptions import ClientError
import aiohttp
from lbry import utils, __version__
if typing.TYPE_CHECKING:
from lbry.conf import Config
LOGGLY_TOKEN = 'BQEzZmMzLJHgAGxkBF00LGD0YGuyATVgAmqxAQEuAQZ2BQH4'
@@ -38,19 +36,17 @@ class JsonFormatter(logging.Formatter):
class HTTPSLogglyHandler(logging.Handler):
def __init__(self, loggly_token: str, config: 'Config'):
def __init__(self, loggly_token: str, fqdn=False, localname=None, facility=None, cookies=None):
super().__init__()
self.cookies = {}
self.fqdn = fqdn
self.localname = localname
self.facility = facility
self.cookies = cookies or {}
self.url = "https://logs-01.loggly.com/inputs/{token}/tag/{tag}".format(
token=utils.deobfuscate(loggly_token), tag='lbrynet-' + __version__
)
self._loop = asyncio.get_event_loop()
self._session = aiohttp.ClientSession()
self._config = config
@property
def enabled(self):
return self._config.share_usage_data
@staticmethod
def get_full_message(record):
@@ -66,14 +62,12 @@ class HTTPSLogglyHandler(logging.Handler):
cookies=self.cookies) as response:
self.cookies.update(response.cookies)
except ClientError:
if self._loop.is_running() and retry and self.enabled:
if self._loop.is_running() and retry:
await self._session.close()
self._session = aiohttp.ClientSession()
return await self._emit(record, retry=False)
def emit(self, record):
if not self.enabled:
return
try:
asyncio.ensure_future(self._emit(record), loop=self._loop)
except RuntimeError: # TODO: use a second loop
@@ -89,7 +83,7 @@ class HTTPSLogglyHandler(logging.Handler):
pass
def get_loggly_handler(config):
handler = HTTPSLogglyHandler(LOGGLY_TOKEN, config=config)
def get_loggly_handler():
handler = HTTPSLogglyHandler(LOGGLY_TOKEN)
handler.setFormatter(JsonFormatter())
return handler
+4 -8
View File
@@ -112,7 +112,7 @@ def _batched_select(transaction, query, parameters, batch_size=900):
def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
raw_content_fee, fully_reflected):
raw_content_fee):
return {
"rowid": rowid,
"added_on": added_on,
@@ -129,8 +129,7 @@ def _get_lbry_file_stream_dict(rowid, added_on, stream_hash, file_name, download
"saved_file": bool(saved_file),
"content_fee": None if not raw_content_fee else Transaction(
binascii.unhexlify(raw_content_fee)
),
"fully_reflected": fully_reflected
)
}
@@ -139,14 +138,11 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
signed_claims = {}
for (rowid, stream_hash, _, file_name, download_dir, data_rate, status, saved_file, raw_content_fee,
added_on, _, sd_hash, stream_key, stream_name, suggested_file_name, *claim_args) in transaction.execute(
"select file.rowid, file.*, stream.*, c.*, "
" case when (SELECT 1 FROM reflected_stream r WHERE r.sd_hash=stream.sd_hash) "
" is null then 0 else 1 end as fully_reflected "
"select file.rowid, file.*, stream.*, c.* "
"from file inner join stream on file.stream_hash=stream.stream_hash "
"inner join content_claim cc on file.stream_hash=cc.stream_hash "
"inner join claim c on cc.claim_outpoint=c.claim_outpoint "
"order by c.rowid desc").fetchall():
claim_args, fully_reflected = tuple(claim_args[:-1]), claim_args[-1]
claim = StoredContentClaim(*claim_args)
if claim.channel_claim_id:
if claim.channel_claim_id not in signed_claims:
@@ -156,7 +152,7 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
_get_lbry_file_stream_dict(
rowid, added_on, stream_hash, file_name, download_dir, data_rate, status,
sd_hash, stream_key, stream_name, suggested_file_name, claim, saved_file,
raw_content_fee, fully_reflected
raw_content_fee
)
)
for claim_name, claim_id in _batched_select(
+13 -15
View File
@@ -19,14 +19,13 @@ class VideoFileAnalyzer:
self._available_encoders = ""
self._ffmpeg_installed = False
self._which = None
self._checked_ffmpeg = False
async def _execute(self, command, arguments):
args = shlex.split(arguments)
process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
stdout, stderr = await process.communicate() # returns when the streams are closed
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
return stdout.decode() + stderr.decode(), process.returncode
async def _verify_executable(self, name):
try:
@@ -49,20 +48,18 @@ class VideoFileAnalyzer:
self._ffmpeg_installed = True
log.debug("Using %s at %s", version.splitlines()[0].split(" Copyright")[0], self._which)
async def status(self, reset=False, recheck=False):
async def status(self, reset=False):
if reset:
self._available_encoders = ""
self._ffmpeg_installed = False
self._which = None
if self._checked_ffmpeg and not recheck:
installed = self._ffmpeg_installed
else:
installed = True
try:
await self._verify_ffmpeg_installed()
except FileNotFoundError:
installed = False
self._checked_ffmpeg = True
installed = True
try:
await self._verify_ffmpeg_installed()
except FileNotFoundError:
installed = False
return {
"available": installed,
"which": self._which,
@@ -122,9 +119,10 @@ class VideoFileAnalyzer:
return ""
result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"')
match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
if match and int(match.group(1)) != 0:
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
iterator = re.finditer(r"\s+seeks:(\d+)\s+", result)
for match in iterator:
if int(match.group(1)) != 0:
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
return ""
@staticmethod
+31 -36
View File
@@ -13,16 +13,6 @@ NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
def set_reference(reference, claim_hash, rows):
if claim_hash:
for txo in rows:
if claim_hash == txo['claim_hash']:
reference.tx_hash = txo['txo_hash'][:32]
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
reference.height = txo['height']
return
class Censor:
__slots__ = 'streams', 'channels', 'censored', 'total'
@@ -49,12 +39,12 @@ class Censor:
self.total += 1
return was_censored
def to_message(self, outputs: OutputsMessage, extra_txo_rows):
def to_message(self, outputs: OutputsMessage):
outputs.blocked_total = self.total
for censoring_channel_hash, count in self.censored.items():
blocked = outputs.blocked.add()
blocked.count = count
set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows)
block = outputs.blocked.add()
block.count = count
block.channel_hash = censoring_channel_hash
class Outputs:
@@ -76,35 +66,26 @@ class Outputs:
for txo_message in self.extra_txos:
self.message_to_txo(txo_message, tx_map)
txos = [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos]
return txos, self.inflate_blocked(tx_map)
return txos, self.inflate_blocked()
def inflate_blocked(self, tx_map):
def inflate_blocked(self):
return {
"total": self.blocked_total,
"channels": [{
'channel': self.message_to_txo(blocked.channel, tx_map),
'blocked': blocked.count
} for blocked in self.blocked]
"channels": {
hexlify(message.channel_hash[::-1]).decode(): message.count
for message in self.blocked
}
}
def message_to_txo(self, txo_message, tx_map):
if txo_message.WhichOneof('meta') == 'error':
error = {
return {
'error': {
'name': txo_message.error.Code.Name(txo_message.error.code),
'name': txo_message.error.Code.Name(txo_message.error.code).lower(),
'text': txo_message.error.text,
}
}
if error['error']['name'] == BLOCKED:
error['error']['censor'] = self.message_to_txo(
txo_message.error.blocked.channel, tx_map
)
return error
tx = tx_map.get(txo_message.tx_hash)
if not tx:
return
txo = tx.outputs[txo_message.nout]
txo = tx_map[txo_message.tx_hash].outputs[txo_message.nout]
if txo_message.WhichOneof('meta') == 'claim':
claim = txo_message.claim
txo.meta = {
@@ -164,7 +145,7 @@ class Outputs:
if total is not None:
page.total = total
if blocked is not None:
blocked.to_message(page, extra_txo_rows)
blocked.to_message(page)
for row in txo_rows:
cls.row_to_message(row, page.txos.add(), extra_txo_rows)
for row in extra_txo_rows:
@@ -181,7 +162,6 @@ class Outputs:
txo_message.error.code = ErrorMessage.NOT_FOUND
elif isinstance(txo, ResolveCensoredError):
txo_message.error.code = ErrorMessage.BLOCKED
set_reference(txo_message.error.blocked.channel, txo.censor_hash, extra_txo_rows)
return
txo_message.tx_hash = txo['txo_hash'][:32]
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
@@ -204,5 +184,20 @@ class Outputs:
txo_message.claim.trending_mixed = txo['trending_mixed']
txo_message.claim.trending_local = txo['trending_local']
txo_message.claim.trending_global = txo['trending_global']
set_reference(txo_message.claim.channel, txo['channel_hash'], extra_txo_rows)
set_reference(txo_message.claim.repost, txo['reposted_claim_hash'], extra_txo_rows)
cls.set_reference(txo_message, 'channel', txo['channel_hash'], extra_txo_rows)
cls.set_reference(txo_message, 'repost', txo['reposted_claim_hash'], extra_txo_rows)
@staticmethod
def set_blocked(message, blocked):
message.blocked_total = blocked.total
@staticmethod
def set_reference(message, attr, claim_hash, rows):
if claim_hash:
for txo in rows:
if claim_hash == txo['claim_hash']:
reference = getattr(message.claim, attr)
reference.tx_hash = txo['txo_hash'][:32]
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
reference.height = txo['height']
break
+5 -6
View File
@@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
name='result.proto',
package='pb',
syntax='proto3',
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.Outputb\x06proto3')
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\".\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x14\n\x0c\x63hannel_hash\x18\x02 \x01(\x0c\x62\x06proto3')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -388,9 +388,9 @@ _BLOCKED = _descriptor.Descriptor(
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='channel', full_name='pb.Blocked.channel', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
name='channel_hash', full_name='pb.Blocked.channel_hash', index=1,
number=2, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
@@ -407,7 +407,7 @@ _BLOCKED = _descriptor.Descriptor(
oneofs=[
],
serialized_start=884,
serialized_end=937,
serialized_end=930,
)
_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
@@ -426,7 +426,6 @@ _CLAIMMETA.fields_by_name['repost'].message_type = _OUTPUT
_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE
_ERROR.fields_by_name['blocked'].message_type = _BLOCKED
_ERROR_CODE.containing_type = _ERROR
_BLOCKED.fields_by_name['channel'].message_type = _OUTPUT
DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS
DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT
DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA
+1 -1
View File
@@ -7,7 +7,7 @@ def _create_url_regex():
# see https://spec.lbry.com/ and test_url.py
invalid_names_regex = \
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+"
r"\u0000-\u0008\u000b-\u000c\u000e-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]+"
def _named(name, regex):
return "(?P<" + name + ">" + regex + ")"
+2 -6
View File
@@ -111,10 +111,6 @@ class ManagedStream:
self.started_writing = asyncio.Event(loop=self.loop)
self.finished_write_attempt = asyncio.Event(loop=self.loop)
@property
def is_fully_reflected(self) -> bool:
return self.fully_reflected.is_set()
@property
def descriptor(self) -> StreamDescriptor:
return self.downloader.descriptor
@@ -433,9 +429,9 @@ class ManagedStream:
await self.loop.create_connection(lambda: protocol, host, port)
await protocol.send_handshake()
sent_sd, needed = await protocol.send_descriptor()
if sent_sd: # reflector needed the sd blob
if sent_sd:
sent.append(self.sd_hash)
if not sent_sd and not needed: # reflector already has the stream
if not sent_sd and not needed:
if not self.fully_reflected.is_set():
self.fully_reflected.set()
await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}")
+2 -4
View File
@@ -116,7 +116,7 @@ class StreamManager:
async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str],
download_directory: Optional[str], status: str,
claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'],
added_on: Optional[int], fully_reflected: bool):
added_on: Optional[int]):
try:
descriptor = await self.blob_manager.get_stream_descriptor(sd_hash)
except InvalidStreamDescriptorError as err:
@@ -127,8 +127,6 @@ class StreamManager:
claim, content_fee=content_fee, rowid=rowid, descriptor=descriptor,
analytics_manager=self.analytics_manager, added_on=added_on
)
if fully_reflected:
stream.fully_reflected.set()
self.streams[sd_hash] = stream
self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream)
@@ -161,7 +159,7 @@ class StreamManager:
file_info['rowid'], file_info['sd_hash'], file_name,
download_directory, file_info['status'],
file_info['claim'], file_info['content_fee'],
file_info['added_on'], file_info['fully_reflected']
file_info['added_on']
)))
if add_stream_tasks:
await asyncio.gather(*add_stream_tasks, loop=self.loop)
-5
View File
@@ -253,11 +253,6 @@ class IntegrationTestCase(AsyncioTestCase):
lambda e: e.tx.id == txid
)
def on_address_update(self, address):
return self.ledger.on_transaction.where(
lambda e: e.address == address
)
def on_transaction_address(self, tx, address):
return self.ledger.on_transaction.where(
lambda e: e.tx.id == tx.id and e.address == address
+16 -16
View File
@@ -1,6 +1,7 @@
import logging
import asyncio
import sqlite3
import sys
from binascii import hexlify
from concurrent.futures.thread import ThreadPoolExecutor
@@ -49,6 +50,11 @@ class AIOSQLite:
def executescript(self, script: str) -> Awaitable:
return self.run(lambda conn: conn.executescript(script))
def create_function(self, name, num_params, func, deterministic=True) -> Awaitable:
if sys.version_info < (3, 8):
return self.run(lambda conn: conn.create_function(name, num_params, func))
return self.run(lambda conn: conn.create_function(name, num_params, func, deterministic=deterministic))
def execute_fetchall(self, sql: str, parameters: Iterable = None) -> Awaitable[Iterable[sqlite3.Row]]:
parameters = parameters if parameters is not None else []
return self.run(lambda conn: conn.execute(sql, parameters).fetchall())
@@ -134,20 +140,16 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
col, op = col[:-len('__not_like')], 'NOT LIKE'
elif key.endswith('__in') or key.endswith('__not_in'):
if key.endswith('__in'):
col, op, one_val_op = col[:-len('__in')], 'IN', '='
col, op = col[:-len('__in')], 'IN'
else:
col, op, one_val_op = col[:-len('__not_in')], 'NOT IN', '!='
col, op = col[:-len('__not_in')], 'NOT IN'
if constraint:
if isinstance(constraint, (list, set, tuple)):
if len(constraint) == 1:
values[f'{key}{tag}'] = next(iter(constraint))
sql.append(f'{col} {one_val_op} :{key}{tag}')
else:
keys = []
for i, val in enumerate(constraint):
keys.append(f':{key}{tag}_{i}')
values[f'{key}{tag}_{i}'] = val
sql.append(f'{col} {op} ({", ".join(keys)})')
keys = []
for i, val in enumerate(constraint):
keys.append(f':{key}{tag}_{i}')
values[f'{key}{tag}_{i}'] = val
sql.append(f'{col} {op} ({", ".join(keys)})')
elif isinstance(constraint, str):
sql.append(f'{col} {op} ({constraint})')
else:
@@ -743,11 +745,9 @@ class Database(SQLiteMixin):
@staticmethod
def constrain_claims(constraints):
claim_types = constraints.pop('claim_type', None)
if isinstance(claim_types, str) and claim_types:
claim_types = [claim_types]
if isinstance(claim_types, list) and claim_types:
constraints['txo_type__in'] = [TXO_TYPES[ct] for ct in claim_types]
claim_type = constraints.pop('claim_type', None)
if claim_type is not None:
constraints['txo_type'] = TXO_TYPES[claim_type]
else:
constraints['txo_type__in'] = CLAIM_TYPES
+1 -2
View File
@@ -630,8 +630,7 @@ class Ledger(metaclass=LedgerRegistry):
raise asyncio.TimeoutError('Timed out waiting for transaction.')
async def _inflate_outputs(self, query, accounts) -> Tuple[List[Output], dict, int, int]:
encoded_outputs = await query
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
outputs = Outputs.from_base64(await query)
txs = []
if len(outputs.txs) > 0:
txs: List[Transaction] = await asyncio.gather(*(
+2 -10
View File
@@ -1,6 +1,5 @@
import logging
import asyncio
import json
from time import perf_counter
from operator import itemgetter
from typing import Dict, Optional, Tuple
@@ -69,14 +68,7 @@ class ClientSession(BaseClientSession):
log.info("timeout sending %s to %s:%i", method, *self.server)
raise asyncio.TimeoutError
if done:
try:
return request.result()
except ConnectionResetError:
log.error(
"wallet server (%s) reset connection upon our %s request, json of %i args is %i bytes",
self.server[0], method, len(args), len(json.dumps(args))
)
raise
return request.result()
except (RPCError, ProtocolError) as e:
log.warning("Wallet server (%s:%i) returned an error. Code: %s Message: %s",
*self.server, *e.args)
@@ -156,7 +148,7 @@ class ClientSession(BaseClientSession):
class Network:
PROTOCOL_VERSION = __version__
MINIMUM_REQUIRED = (0, 59, 0)
MINIMUM_REQUIRED = (0, 57, 0)
def __init__(self, ledger):
self.ledger = ledger
-5
View File
@@ -173,9 +173,6 @@ class SPVNode:
async def start(self, blockchain_node: 'BlockchainNode', extraconf=None):
self.data_path = tempfile.mkdtemp()
conf = {
'DESCRIPTION': '',
'PAYMENT_ADDRESS': '',
'DAILY_FEE': '0',
'DB_DIRECTORY': self.data_path,
'DAEMON_URL': blockchain_node.rpc_url,
'REORG_LIMIT': '100',
@@ -231,7 +228,6 @@ class BlockchainProcess(asyncio.SubprocessProtocol):
def process_exited(self):
self.stopped.set()
self.ready.set()
class BlockchainNode:
@@ -319,7 +315,6 @@ class BlockchainNode:
BlockchainProcess, *command
)
await self.protocol.ready.wait()
assert not self.protocol.stopped.is_set()
async def stop(self, cleanup=True):
try:
+3 -5
View File
@@ -386,11 +386,9 @@ class RPCSession(SessionBase):
while not self.is_closing():
try:
message = await self.framer.receive_message()
except MemoryError:
self.logger.warning('received oversized message from %s:%s, dropping connection',
self._address[0], self._address[1])
self._close()
return
except MemoryError as e:
self.logger.warning(f'{e!r}')
continue
self.last_recv = time.perf_counter()
self.recv_count += 1
+3 -3
View File
@@ -164,7 +164,6 @@ class BlockProcessor:
# is consistent with self.height
self.state_lock = asyncio.Lock()
self.search_cache = {}
async def run_in_thread_with_lock(self, func, *args):
# Run in a thread to prevent blocking. Shielded so that
# cancellations from shutdown don't lose work - when the task
@@ -192,8 +191,6 @@ class BlockProcessor:
if hprevs == chain:
start = time.perf_counter()
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
for cache in self.search_cache.values():
cache.clear()
await self._maybe_flush()
processed_time = time.perf_counter() - start
BLOCK_COUNT.set(self.height)
@@ -732,6 +729,7 @@ class LBRYBlockProcessor(BlockProcessor):
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
self.sql: SQLDB = self.db.sql
self.timer = Timer('BlockProcessor')
self.search_cache = {}
def advance_blocks(self, blocks):
self.sql.begin()
@@ -746,6 +744,8 @@ class LBRYBlockProcessor(BlockProcessor):
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
if self.env.individual_tag_indexes:
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
for cache in self.search_cache.values():
cache.clear()
def advance_txs(self, height, txs, header):
timer = self.timer.sub_timers['advance_blocks']
-1
View File
@@ -1,7 +1,6 @@
CLAIM_TYPES = {
'stream': 1,
'channel': 2,
'repost': 3
}
STREAM_TYPES = {
+17 -21
View File
@@ -4,11 +4,10 @@ import apsw
import logging
from operator import itemgetter
from typing import Tuple, List, Dict, Union, Type, Optional
from binascii import unhexlify
from binascii import unhexlify, hexlify
from decimal import Decimal
from contextvars import ContextVar
from functools import wraps
from itertools import chain
from dataclasses import dataclass
from lbry.wallet.database import query, interpolate
@@ -266,7 +265,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
else:
constraints['claim.claim_id__like'] = f'{claim_id[:40]}%'
elif 'claim_ids' in constraints:
constraints['claim.claim_id__in'] = set(constraints.pop('claim_ids'))
constraints['claim.claim_id__in'] = constraints.pop('claim_ids')
if 'reposted_claim_id' in constraints:
constraints['claim.reposted_claim_hash'] = unhexlify(constraints.pop('reposted_claim_id'))[::-1]
@@ -282,15 +281,15 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
if 'channel_ids' in constraints:
channel_ids = constraints.pop('channel_ids')
if channel_ids:
constraints['claim.channel_hash__in'] = {
constraints['claim.channel_hash__in'] = [
unhexlify(cid)[::-1] for cid in channel_ids
}
]
if 'not_channel_ids' in constraints:
not_channel_ids = constraints.pop('not_channel_ids')
if not_channel_ids:
not_channel_ids_binary = {
not_channel_ids_binary = [
unhexlify(ncid)[::-1] for ncid in not_channel_ids
}
]
if constraints.get('has_channel_signature', False):
constraints['claim.channel_hash__not_in'] = not_channel_ids_binary
else:
@@ -320,13 +319,13 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
if 'stream_types' in constraints:
stream_types = constraints.pop('stream_types')
if stream_types:
constraints['claim.stream_type__in'] = {
constraints['claim.stream_type__in'] = [
STREAM_TYPES[stream_type] for stream_type in stream_types
}
]
if 'media_types' in constraints:
media_types = constraints.pop('media_types')
if media_types:
constraints['claim.media_type__in'] = set(media_types)
constraints['claim.media_type__in'] = media_types
if 'fee_currency' in constraints:
constraints['claim.fee_currency'] = constraints.pop('fee_currency').lower()
@@ -389,13 +388,9 @@ def search_claims(censor: Censor, **constraints) -> List:
)
def _get_referenced_rows(txo_rows: List[dict], censor_channels: List[bytes]):
censor = ctx.get().get_resolve_censor()
def _get_referenced_rows(censor: Censor, txo_rows: List[dict]):
repost_hashes = set(filter(None, map(itemgetter('reposted_claim_hash'), txo_rows)))
channel_hashes = set(chain(
filter(None, map(itemgetter('channel_hash'), txo_rows)),
censor_channels
))
channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows)))
reposted_txos = []
if repost_hashes:
@@ -418,10 +413,12 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
total = count_claims(**constraints)
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
if 'order_by' not in constraints:
constraints['order_by'] = ["claim_hash"]
context = ctx.get()
search_censor = context.get_search_censor()
txo_rows = search_claims(search_censor, **constraints)
extra_txo_rows = _get_referenced_rows(txo_rows, search_censor.censored.keys())
extra_txo_rows = _get_referenced_rows(context.get_resolve_censor(), txo_rows)
return txo_rows, extra_txo_rows, constraints['offset'], total, search_censor
@@ -429,8 +426,7 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
def resolve(urls) -> Tuple[List, List]:
txo_rows = [resolve_url(raw_url) for raw_url in urls]
extra_txo_rows = _get_referenced_rows(
[txo for txo in txo_rows if isinstance(txo, dict)],
[txo.censor_hash for txo in txo_rows if isinstance(txo, ResolveCensoredError)]
ctx.get().get_resolve_censor(), [r for r in txo_rows if isinstance(r, dict)]
)
return txo_rows, extra_txo_rows
@@ -456,7 +452,7 @@ def resolve_url(raw_url):
if matches:
channel = matches[0]
elif censor.censored:
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
return ResolveCensoredError(raw_url, hexlify(next(iter(censor.censored))[::-1]).decode())
else:
return LookupError(f'Could not find channel in "{raw_url}".')
@@ -476,7 +472,7 @@ def resolve_url(raw_url):
if matches:
return matches[0]
elif censor.censored:
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
return ResolveCensoredError(raw_url, hexlify(next(iter(censor.censored))[::-1]).decode())
else:
return LookupError(f'Could not find claim at "{raw_url}".')
+22 -26
View File
@@ -24,23 +24,16 @@ DECAY_PER_RENORM = DECAY**(RENORM_INTERVAL)
TRENDING_LOG = True
# Stubs
def install(connection):
"""
Install the AR trending algorithm.
"""
check_trending_values(connection)
if TRENDING_LOG:
f = open("trending_ar.log", "a")
f.close()
# Stub
CREATE_TREND_TABLE = ""
def check_trending_values(connection):
"""
If the trending values appear to be based on the zscore algorithm,
If the trending values appear to be based on the standard algorithm,
reset them. This will allow resyncing from a standard snapshot.
"""
c = connection.cursor()
@@ -60,6 +53,9 @@ def check_trending_values(connection):
COMMIT;""")
print("done.")
# Create the index
c.execute("create index if not exists claim_id_only_idx on claim (claim_id);")
def spike_height(trending_score, x, x_old, time_boost=1.0):
"""
@@ -75,9 +71,8 @@ def spike_height(trending_score, x, x_old, time_boost=1.0):
# Softened change in amount counts more for minnows
if delta > 0.0:
if trending_score >= 0.0:
multiplier = 0.1/((trending_score/time_boost + softened_change_in_amount) + 1.0)
softened_change_in_amount *= multiplier
multiplier = 1.0/math.sqrt(x + 1.0)
softened_change_in_amount *= multiplier
else:
softened_change_in_amount *= -1.0
@@ -111,14 +106,14 @@ class TrendingData:
# Have all claims been read from db yet?
self.initialised = False
def insert_claim_from_load(self, claim_hash, trending_score, total_amount):
def insert_claim_from_load(self, claim_id, trending_score, total_amount):
assert not self.initialised
self.claims[claim_hash] = {"trending_score": trending_score,
"total_amount": total_amount,
"changed": False}
self.claims[claim_id] = {"trending_score": trending_score,
"total_amount": total_amount,
"changed": False}
def update_claim(self, claim_hash, total_amount, time_boost=1.0):
def update_claim(self, claim_id, total_amount, time_boost=1.0):
"""
Update trending data for a claim, given its new total amount.
"""
@@ -126,8 +121,8 @@ class TrendingData:
# Extract existing total amount and trending score
# or use starting values if the claim is new
if claim_hash in self.claims:
old_state = copy.deepcopy(self.claims[claim_hash])
if claim_id in self.claims:
old_state = copy.deepcopy(self.claims[claim_id])
else:
old_state = {"trending_score": 0.0,
"total_amount": 0.0,
@@ -143,9 +138,9 @@ class TrendingData:
old_state["total_amount"],
time_boost)
trending_score = old_state["trending_score"] + spike
self.claims[claim_hash] = {"total_amount": total_amount,
"trending_score": trending_score,
"changed": True}
self.claims[claim_id] = {"total_amount": total_amount,
"trending_score": trending_score,
"changed": True}
@@ -175,12 +170,13 @@ def test_trending():
# One global instance
# pylint: disable=C0103
trending_data = TrendingData()
f = open("trending_ar.log", "w")
f.close()
def run(db, height, final_height, recalculate_claim_hashes):
if height < final_height - 5*HALF_LIFE:
trending_log("Skipping AR trending at block {h}.\n".format(h=height))
return
start = time.time()
@@ -216,7 +212,7 @@ def run(db, height, final_height, recalculate_claim_hashes):
if not trending_data.initialised:
# On fresh launch
for row in db.execute("""
SELECT claim_hash, trending_mixed,
SELECT claim_id, trending_mixed,
(amount + support_amount)
AS total_amount
FROM claim;
@@ -225,7 +221,7 @@ def run(db, height, final_height, recalculate_claim_hashes):
trending_data.initialised = True
else:
for row in db.execute(f"""
SELECT claim_hash,
SELECT claim_id,
(amount + support_amount)
AS total_amount
FROM claim
@@ -252,7 +248,7 @@ def run(db, height, final_height, recalculate_claim_hashes):
trending_log("{n} scores to write...".format(n=len(the_list)))
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;",
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_id=?;",
the_list)
trending_log("done.\n")
+21 -30
View File
@@ -94,7 +94,6 @@ class SQLDB:
create index if not exists claim_txo_hash_idx on claim (txo_hash);
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
create index if not exists claim_expiration_height_idx on claim (expiration_height);
create index if not exists claim_reposted_claim_hash_idx on claim (reposted_claim_hash);
"""
CREATE_SUPPORT_TABLE = """
@@ -130,42 +129,35 @@ class SQLDB:
-- used by any tag clouds
create index if not exists tag_tag_idx on tag (tag, claim_hash);
-- naked order bys (no filters)
create unique index if not exists claim_release_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_idx on claim (trending_group, trending_mixed, claim_hash);
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash);
-- common ORDER BY
create unique index if not exists claim_effective_amount_idx on claim (effective_amount, claim_hash, release_time);
create unique index if not exists claim_release_time_idx on claim (release_time, claim_hash);
create unique index if not exists claim_trending_global_mixed_idx on claim (trending_global, trending_mixed, claim_hash);
create unique index if not exists claim_trending_group_mixed_idx on claim (trending_group, trending_mixed, claim_hash);
create unique index if not exists filter_fee_amount_order_release_time_idx on claim (fee_amount, release_time, claim_hash);
-- claim_type filter + order by
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_global, trending_mixed, claim_hash);
create unique index if not exists claim_type_release_idx on claim (claim_type, release_time, claim_hash);
create unique index if not exists claim_type_trending_idx on claim (claim_type, trending_group, trending_mixed, claim_hash);
create unique index if not exists claim_type_effective_amount_idx on claim (claim_type, effective_amount, claim_hash);
-- stream_type filter + order by
create unique index if not exists stream_type_release_idx on claim (stream_type, release_time, claim_hash);
create unique index if not exists stream_type_trending_idx on claim (stream_type, trending_group, trending_mixed, claim_hash);
create unique index if not exists stream_type_effective_amount_idx on claim (stream_type, effective_amount, claim_hash);
create unique index if not exists channel_hash_release_time_idx on claim (channel_hash, release_time, claim_hash);
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, trending_global, trending_mixed, claim_hash);
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, effective_amount, claim_hash);
-- channel_hash filter + order by
create unique index if not exists channel_hash_release_idx on claim (channel_hash, release_time, claim_hash);
create unique index if not exists channel_hash_trending_idx on claim (channel_hash, trending_group, trending_mixed, claim_hash);
create unique index if not exists channel_hash_effective_amount_idx on claim (channel_hash, effective_amount, claim_hash);
-- duration filter + order by
create unique index if not exists duration_release_idx on claim (duration, release_time, claim_hash);
create unique index if not exists duration_trending_idx on claim (duration, trending_group, trending_mixed, claim_hash);
create unique index if not exists duration_effective_amount_idx on claim (duration, effective_amount, claim_hash);
-- fee_amount + order by
create unique index if not exists fee_amount_release_idx on claim (fee_amount, release_time, claim_hash);
create unique index if not exists fee_amount_trending_idx on claim (fee_amount, trending_group, trending_mixed, claim_hash);
create unique index if not exists fee_amount_effective_amount_idx on claim (fee_amount, effective_amount, claim_hash);
create unique index if not exists filter_stream_duration_idx on claim (duration, trending_global, trending_mixed, claim_hash);
-- TODO: verify that all indexes below are used
create index if not exists claim_height_normalized_idx on claim (height, normalized asc);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_id_idx on claim (claim_id, claim_hash);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
"""
@@ -249,10 +241,10 @@ class SQLDB:
streams, channels = {}, {}
if channel_hashes:
sql = query(
"SELECT repost.channel_hash, repost.reposted_claim_hash, target.claim_type "
"FROM claim as repost JOIN claim AS target ON (target.claim_hash=repost.reposted_claim_hash)", **{
'repost.reposted_claim_hash__is_not_null': 1,
'repost.channel_hash__in': channel_hashes
"SELECT claim.channel_hash, claim.reposted_claim_hash, reposted.claim_type "
"FROM claim JOIN claim AS reposted ON (reposted.claim_hash=claim.reposted_claim_hash)", **{
'claim.reposted_claim_hash__is_not_null': 1,
'claim.channel_hash__in': channel_hashes
}
)
for blocked_claim in self.execute(*sql):
@@ -368,7 +360,6 @@ class SQLDB:
if isinstance(fee.amount, Decimal):
claim_record['fee_amount'] = int(fee.amount*1000)
elif claim.is_repost:
claim_record['claim_type'] = CLAIM_TYPES['repost']
claim_record['reposted_claim_hash'] = claim.repost.reference.claim_hash
elif claim.is_channel:
claim_record['claim_type'] = CLAIM_TYPES['channel']
+1 -11
View File
@@ -72,18 +72,16 @@ class Env:
self.tor_proxy_host = self.default('TOR_PROXY_HOST', 'localhost')
self.tor_proxy_port = self.integer('TOR_PROXY_PORT', None)
# The electrum client takes the empty string as unspecified
self.payment_address = self.default('PAYMENT_ADDRESS', '')
self.donation_address = self.default('DONATION_ADDRESS', '')
# Server limits to help prevent DoS
self.max_send = self.integer('MAX_SEND', 1000000)
self.max_receive = self.integer('MAX_RECEIVE', 1000000)
self.max_subs = self.integer('MAX_SUBS', 250000)
self.max_sessions = self.sane_max_sessions()
self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000)
self.session_timeout = self.integer('SESSION_TIMEOUT', 600)
self.drop_client = self.custom("DROP_CLIENT", None, re.compile)
self.description = self.default('DESCRIPTION', '')
self.daily_fee = self.string_amount('DAILY_FEE', '0')
self.daily_fee = self.integer('DAILY_FEE', 0)
# Identities
clearnet_identity = self.clearnet_identity()
@@ -109,14 +107,6 @@ class Env:
raise cls.Error(f'required envvar {envvar} not set')
return value
@classmethod
def string_amount(cls, envvar, default):
value = environ.get(envvar, default)
amount_pattern = re.compile("[0-9]{0,10}(\.[0-9]{1,8})?")
if len(value) > 0 and not amount_pattern.fullmatch(value):
raise cls.Error(f'{value} is not a valid amount for {envvar}')
return value
@classmethod
def integer(cls, envvar, default):
value = environ.get(envvar)
+2 -13
View File
@@ -30,7 +30,6 @@ from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
from lbry.wallet.server.prometheus import REQUESTS_COUNT, SQLITE_INTERRUPT_COUNT, SQLITE_INTERNAL_ERROR_COUNT
from lbry.wallet.server.prometheus import SQLITE_OPERATIONAL_ERROR_COUNT, SQLITE_EXECUTOR_TIMES, SESSIONS_COUNT
from lbry.wallet.server.prometheus import SQLITE_PENDING_COUNT, CLIENT_VERSIONS
from lbry.wallet.rpc.framing import NewlineFramer
import lbry.wallet.server.version as VERSION
from lbry.wallet.rpc import (
@@ -622,7 +621,6 @@ class SessionBase(RPCSession):
def __init__(self, session_mgr, db, mempool, peer_mgr, kind):
connection = JSONRPCConnection(JSONRPCAutoDetect)
self.env = session_mgr.env
super().__init__(connection=connection)
self.logger = util.class_logger(__name__, self.__class__.__name__)
self.session_mgr = session_mgr
@@ -630,6 +628,7 @@ class SessionBase(RPCSession):
self.mempool = mempool
self.peer_mgr = peer_mgr
self.kind = kind # 'RPC', 'TCP' etc.
self.env = session_mgr.env
self.coin = self.env.coin
self.anon_logs = self.env.anon_logs
self.txs_sent = 0
@@ -642,9 +641,6 @@ class SessionBase(RPCSession):
async def notify(self, touched, height_changed):
pass
def default_framer(self):
return NewlineFramer(self.env.max_receive)
def peer_address_str(self, *, for_log=True):
"""Returns the peer's IP address and port as a human-readable
string, respecting anon logs if the output is for a log."""
@@ -804,7 +800,6 @@ class LBRYElectrumX(SessionBase):
'blockchain.transaction.get_merkle': cls.transaction_merkle,
'server.add_peer': cls.add_peer,
'server.banner': cls.banner,
'server.payment_address': cls.payment_address,
'server.donation_address': cls.donation_address,
'server.features': cls.server_features_async,
'server.peers.subscribe': cls.peers_subscribe,
@@ -859,8 +854,7 @@ class LBRYElectrumX(SessionBase):
'protocol_max': max_str,
'genesis_hash': env.coin.GENESIS_HASH,
'description': env.description,
'payment_address': env.payment_address,
'donation_address': env.donation_address,
'payment_address': env.donation_address,
'daily_fee': env.daily_fee,
'hash_function': 'sha256',
}
@@ -1368,16 +1362,11 @@ class LBRYElectrumX(SessionBase):
('$SERVER_VERSION', self.version),
('$DAEMON_VERSION', daemon_version),
('$DAEMON_SUBVERSION', network_info['subversion']),
('$PAYMENT_ADDRESS', self.env.payment_address),
('$DONATION_ADDRESS', self.env.donation_address),
]:
banner = banner.replace(*pair)
return banner
async def payment_address(self):
"""Return the payment address as a string, empty if there is none."""
return self.env.payment_address
async def donation_address(self):
"""Return the donation address as a string, empty if there is none."""
return self.env.donation_address
-78
View File
@@ -1,78 +0,0 @@
import asyncio
import logging
from lbry.error import (
ServerPaymentFeeAboveMaxAllowedError,
ServerPaymentInvalidAddressError,
ServerPaymentWalletLockedError
)
from lbry.wallet.dewies import lbc_to_dewies
from lbry.wallet.stream import StreamController
from lbry.wallet.transaction import Output, Transaction
log = logging.getLogger(__name__)
class WalletServerPayer:
def __init__(self, payment_period=24 * 60 * 60, max_fee='1.0', analytics_manager=None):
self.ledger = None
self.wallet = None
self.running = False
self.task = None
self.payment_period = payment_period
self.analytics_manager = analytics_manager
self.max_fee = max_fee
self._on_payment_controller = StreamController()
self.on_payment = self._on_payment_controller.stream
self.on_payment.listen(None, on_error=lambda e: logging.warning(e.args[0]))
async def pay(self):
while self.running:
await asyncio.sleep(self.payment_period)
features = await self.ledger.network.retriable_call(self.ledger.network.get_server_features)
address = features['payment_address']
amount = str(features['daily_fee'])
if not address or not amount:
continue
if not self.ledger.is_valid_address(address):
self._on_payment_controller.add_error(ServerPaymentInvalidAddressError(address))
continue
if self.wallet.is_locked:
self._on_payment_controller.add_error(ServerPaymentWalletLockedError())
continue
amount = lbc_to_dewies(features['daily_fee']) # check that this is in lbc and not dewies
limit = lbc_to_dewies(self.max_fee)
if amount > limit:
self._on_payment_controller.add_error(
ServerPaymentFeeAboveMaxAllowedError(features['daily_fee'], self.max_fee)
)
continue
tx = await Transaction.create(
[],
[Output.pay_pubkey_hash(amount, self.ledger.address_to_hash160(address))],
self.wallet.get_accounts_or_all(None),
self.wallet.get_account_or_default(None)
)
await self.ledger.broadcast(tx)
if self.analytics_manager:
await self.analytics_manager.send_credits_sent()
self._on_payment_controller.add(tx)
async def start(self, ledger=None, wallet=None):
if lbc_to_dewies(self.max_fee) < 1:
return
self.ledger = ledger
self.wallet = wallet
self.running = True
self.task = asyncio.ensure_future(self.pay())
self.task.add_done_callback(lambda _: log.info("Stopping wallet server payments."))
async def stop(self):
if self.running:
self.running = False
self.task.cancel()
-20
View File
@@ -1,20 +0,0 @@
#!/usr/bin/env bash
# usage: update_dev_wallet_server.sh <host to update>
TARGET_HOST=$1
SCRIPTS_DIR=`dirname $0`
LBRY_DIR=`dirname $SCRIPTS_DIR`
# build the image
docker build -f $LBRY_DIR/docker/Dockerfile.wallet_server -t lbry/wallet-server:development $LBRY_DIR
IMAGE=`docker image inspect lbry/wallet-server:development | sed -n "s/^.*Id\":\s*\"sha256:\s*\(\S*\)\".*$/\1/p"`
# push the image to the server
ssh $TARGET_HOST docker image prune --force
docker save $IMAGE | ssh $TARGET_HOST docker load
ssh $TARGET_HOST docker tag $IMAGE lbry/wallet-server:development
# restart the wallet server
ssh $TARGET_HOST docker-compose down
ssh $TARGET_HOST WALLET_SERVER_TAG="development" docker-compose up -d
+2 -6
View File
@@ -124,11 +124,7 @@ def release(args):
current_version = Version.from_content(version_file)
print(f'Current Version: {current_version}')
if args.action == 'current':
new_version = current_version
else:
new_version = current_version.increment(args.action)
new_version = current_version.increment(args.action)
print(f' New Version: {new_version}')
previous_release = repo.release_from_tag(args.start_tag or current_version.tag)
@@ -245,7 +241,7 @@ def main():
parser.add_argument("--confirm", default=False, action="store_true",
help="without this flag, it will only print what it will do but will not actually do it")
parser.add_argument("--start-tag", help="custom starting tag for changelog generation")
parser.add_argument("action", choices=['test', 'current', 'major', 'minor', 'micro'])
parser.add_argument("action", choices=['test', 'major', 'minor', 'micro'])
args = parser.parse_args()
if args.action == "test":
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
import asyncio
import logging
import os
import sys
from lbry.wallet import database
def enable_logging():
root = logging.getLogger()
root.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s') # %(asctime)s - %(levelname)s -
handler.setFormatter(formatter)
root.addHandler(handler)
async def initialize_tables(queue, old_file, db):
await db.executescript("PRAGMA journal_mode=WAL")
await db.executescript("PRAGMA case_sensitive_like=true")
await db.executescript("CREATE TABLE IF NOT EXISTS claim_with_metadata "
"(claimID BLOB NOT NULL PRIMARY KEY, "
"name BLOB NOT NULL, "
"originalHeight INTEGER NOT NULL, "
"updateHeight INTEGER NOT NULL, "
"activationHeight INTEGER NOT NULL, "
"metadataField1 TEXT)")
await db.executescript(f"ATTACH '{old_file}' AS crd")
await db.create_function("NEEDS_UPDATE", 5, lambda id, name, original, update, activation:
queue.put_nowait({
"claimID": id, "name": name, "originalHeight": original,
"updateHeight": update, "activationHeight": activation
}))
await db.executescript("CREATE TEMP TRIGGER on_claim_insert AFTER INSERT ON crd.claim "
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(NEW.claimID, NEW.nodeName, "
"NEW.originalHeight, NEW.updateHeight, NEW.ActivationHeight); END")
await db.executescript("CREATE TEMP TRIGGER on_claim_update AFTER UPDATE ON crd.claim "
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(NEW.claimID, NEW.nodeName, "
"NEW.originalHeight, NEW.updateHeight, NEW.ActivationHeight); END")
await db.executescript("CREATE TEMP TRIGGER on_claim_delete AFTER DELETE ON crd.claim "
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(OLD.claimID, OLD.nodeName, 0, 0, 0); END")
async def catch_up_on_missing(queue, db):
# items removed from crd are those that are in claim_with_metadata but not in crd
# items inserted into crd are those in crd but not in claim_with_metadata
# items updated are those that are in both but some columns don't match
await db.executescript("DELETE FROM claim_with_metadata WHERE claimID NOT IN"
"(SELECT c.claimID FROM crd.claim c)")
await db.executescript("INSERT INTO claim_with_metadata(claimID, name, originalHeight, updateHeight, activationHeight) "
"SELECT c.claimID, c.nodeName, c.originalHeight, c.updateHeight, c.activationHeight "
"FROM crd.claim c LEFT JOIN claim_with_metadata m ON c.claimID = m.claimID "
"WHERE m.claimID IS NULL OR c.nodeName != m.name OR c.updateHeight != m.updateHeight "
"OR c.activationHeight != m.activationHeight ON CONFLICT(claimID) DO UPDATE SET "
"name = excluded.name, originalHeight = excluded.originalHeight, "
"updateHeight = excluded.updateHeight, activationHeight = excluded.activationHeight")
async def remove_claim(claim, db):
await db.executescript("DELETE FROM claim_with_metadata WHERE claimID = :claimID", claim)
async def insert_update(claim, db):
# TODO: lookup and parse metadata here
await db.executescript("INSERT INTO claim_with_metadata(claimID, name, originalHeight, updateHeight, activationHeight) "
"VALUES(:claimID, :name, :originalHeight, :updateHeight, :activationHeight) "
"ON CONFLICT(claimID) DO UPDATE SET name = excluded.name, "
"originalHeight = excluded.originalHeight, updateHeight = excluded.updateHeight, "
"activationHeight = excluded.activationHeight", claim)
async def handle_changes():
old_file = os.path.expanduser("~/.lbrycrd/claims.sqlite")
new_file = os.path.expanduser("~/.lbrycrd/duplicate.sqlite")
db = database.AIOSQLite()
db = await db.connect(new_file) # super confusing that you have to use the return value here
queue = asyncio.Queue()
await initialize_tables(queue, old_file, db)
await catch_up_on_missing(queue, db)
while True:
claim = await queue.get()
if claim.updateHeight <= 0:
await remove_claim(claim, db)
else:
await insert_update(claim, db)
def main():
enable_logging()
try:
asyncio.run(handle_changes())
except KeyboardInterrupt:
logging.info("Process interrupted")
if __name__ == '__main__':
main()
+1 -1
View File
@@ -12,7 +12,7 @@ async def main():
try:
hostname = sys.argv[1]
except IndexError:
hostname = 'spv11.lbry.com'
hostname = 'spv1.lbry.com'
loop = asyncio.get_event_loop()
client = ClientSession(network=None, server=(hostname, 50001))
-1
View File
@@ -32,7 +32,6 @@ disable=
too-many-branches,
too-many-arguments,
too-many-statements,
too-many-nested-blocks,
too-many-public-methods,
too-many-instance-attributes,
protected-access,
+1 -1
View File
@@ -40,7 +40,7 @@ setup(
'colorama==0.3.7',
'distro==1.4.0',
'base58==1.0.0',
'cffi==1.13.2',
'cffi==1.12.1',
'cryptography==2.5',
'protobuf==3.6.1',
'msgpack==0.6.1',
@@ -1,10 +1,10 @@
import asyncio
import os.path
import tempfile
import logging
from binascii import unhexlify
from urllib.request import urlopen
from lbry.error import InsufficientFundsError
from lbry.extras.daemon.daemon import DEFAULT_PAGE_SIZE
@@ -73,19 +73,6 @@ class ClaimSearchCommand(ClaimTestCase):
(result['txid'], result['claim_id'])
)
async def test_disconnect_on_memory_error(self):
claim_ids = [
'0000000000000000000000000000000000000000',
] * 23828
self.assertListEqual([], await self.claim_search(claim_ids=claim_ids))
# 23829 claim ids makes the request just large enough
claim_ids = [
'0000000000000000000000000000000000000000',
] * 23829
with self.assertRaises(ConnectionResetError):
await self.claim_search(claim_ids=claim_ids)
async def test_basic_claim_search(self):
await self.create_channel()
channel_txo = self.channel['outputs'][0]
@@ -322,14 +309,12 @@ class ClaimSearchCommand(ClaimTestCase):
octet = await self.stream_create()
video = await self.stream_create('chrome', file_path=self.video_file_name)
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
repost = await self.stream_repost(self.get_claim_id(image))
channel = await self.channel_create()
unknown = self.sout(tx)
# claim_type
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
await self.assertFindsClaims([channel], claim_type='channel')
await self.assertFindsClaims([repost], claim_type='repost')
# stream_type
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
@@ -337,7 +322,7 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([image], stream_types=['image'])
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
# media_type
# stream_type
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
await self.assertFindsClaims([video], media_types=['video/mp4'])
await self.assertFindsClaims([image], media_types=['image/png'])
@@ -399,25 +384,7 @@ class ClaimSearchCommand(ClaimTestCase):
class ClaimCommands(ClaimTestCase):
async def test_claim_list_type_filtering(self):
await self.channel_create()
await self.stream_create()
r = await self.claim_list(claim_type='channel')
self.assertEqual(1, len(r))
self.assertEqual('channel', r[0]['value_type'])
r = await self.claim_list(claim_type='stream')
self.assertEqual(1, len(r))
self.assertEqual('stream', r[0]['value_type'])
r = await self.claim_list(claim_type=['stream', 'channel'])
self.assertEqual(2, len(r))
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
async def test_claim_stream_channel_list_with_resolve(self):
self.assertListEqual([], await self.claim_list(resolve=True))
await self.channel_create()
await self.stream_create()
@@ -438,10 +405,10 @@ class ClaimCommands(ClaimTestCase):
await self.ledger.wait(channel_tx)
r = await self.claim_list(resolve=True)
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
self.assertEqual('not_found', r[0]['meta']['error']['name'])
self.assertTrue(r[1]['meta']['is_controlling'])
r = await self.channel_list(resolve=True)
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
self.assertEqual('not_found', r[0]['meta']['error']['name'])
self.assertTrue(r[1]['meta']['is_controlling'])
# confirm it
@@ -463,10 +430,10 @@ class ClaimCommands(ClaimTestCase):
await self.ledger.wait(stream_tx)
r = await self.claim_list(resolve=True)
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
self.assertEqual('not_found', r[0]['meta']['error']['name'])
self.assertTrue(r[1]['meta']['is_controlling'])
r = await self.stream_list(resolve=True)
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
self.assertEqual('not_found', r[0]['meta']['error']['name'])
self.assertTrue(r[1]['meta']['is_controlling'])
# confirm it
@@ -878,26 +845,18 @@ class StreamCommands(ClaimTestCase):
# search for blocked content directly
result = await self.out(self.daemon.jsonrpc_claim_search(name='bad_content'))
blocked = result['blocked']
self.assertEqual([], result['items'])
self.assertEqual(1, blocked['total'])
self.assertEqual(1, len(blocked['channels']))
self.assertEqual(1, blocked['channels'][0]['blocked'])
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
self.assertEqual({"channels": {filtering_channel_id: 1}, "total": 1}, result['blocked'])
# search channel containing blocked content
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel'))
blocked = result['blocked']
self.assertEqual(1, len(result['items']))
self.assertEqual(1, blocked['total'])
self.assertEqual(1, len(blocked['channels']))
self.assertEqual(1, blocked['channels'][0]['blocked'])
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
self.assertEqual({"channels": {filtering_channel_id: 1}, "total": 1}, result['blocked'])
# content was filtered by not_tag before censoring
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel', not_tags=["good", "bad"]))
self.assertEqual(0, len(result['items']))
self.assertEqual({"channels": [], "total": 0}, result['blocked'])
self.assertEqual({"channels": {}, "total": 0}, result['blocked'])
blocking_channel_id = self.get_claim_id(
await self.channel_create('@blocking', '1.0')
@@ -915,9 +874,8 @@ class StreamCommands(ClaimTestCase):
# blocked content is not resolveable
result = await self.out(self.daemon.jsonrpc_resolve('lbry://@some_channel/bad_content'))
error = result['lbry://@some_channel/bad_content']['error']
self.assertEqual(error['name'], 'BLOCKED')
self.assertTrue(error['name'], 'blocked')
self.assertTrue(error['text'].startswith("Resolve of 'lbry://@some_channel/bad_content' was censored"))
self.assertTrue(error['censor']['short_url'].startswith('lbry://@blocking#'))
async def test_publish_updates_file_list(self):
tx = await self.stream_create(title='created')
+10 -15
View File
@@ -1,4 +1,5 @@
import logging
import os
import asyncio
import lbry
@@ -28,20 +29,15 @@ class NetworkTests(IntegrationTestCase):
'pruning': None,
'description': '',
'payment_address': '',
'donation_address': '',
'daily_fee': '0',
'daily_fee': 0,
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
await self.conductor.spv_node.stop()
payment_address, donation_address = await self.account.get_addresses(limit=2)
await self.conductor.spv_node.start(
self.conductor.blockchain_node,
extraconf={
'DESCRIPTION': 'Fastest server in the west.',
'PAYMENT_ADDRESS': payment_address,
'DONATION_ADDRESS': donation_address,
'DAILY_FEE': '42'
}
)
address = (await self.account.get_addresses(limit=1))[0]
os.environ.update({
'DESCRIPTION': 'Fastest server in the west.',
'DONATION_ADDRESS': address,
'DAILY_FEE': '42'})
await self.conductor.spv_node.start(self.conductor.blockchain_node)
await self.ledger.network.on_connected.first
self.assertDictEqual({
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
@@ -51,9 +47,8 @@ class NetworkTests(IntegrationTestCase):
'protocol_min': '0.54.0',
'pruning': None,
'description': 'Fastest server in the west.',
'payment_address': payment_address,
'donation_address': donation_address,
'daily_fee': '42',
'payment_address': address,
'daily_fee': 42,
'server_version': lbry.__version__}, await self.ledger.network.get_server_features())
@@ -15,7 +15,7 @@ class BaseResolveTestCase(CommandTestCase):
other = (await self.resolve(name))[name]
if claim_id is None:
self.assertIn('error', other)
self.assertEqual(other['error']['name'], 'NOT_FOUND')
self.assertEqual(other['error']['name'], 'not_found')
else:
self.assertEqual(claim_id, other['claim_id'])
@@ -186,7 +186,7 @@ class ResolveCommand(BaseResolveTestCase):
self.assertEqual(response, {
'lbry://@abc/on-channel-claim': {
'error': {
'name': 'NOT_FOUND',
'name': 'not_found',
'text': 'Could not find claim at "lbry://@abc/on-channel-claim".',
}
}
@@ -265,7 +265,7 @@ class ResolveCommand(BaseResolveTestCase):
self.assertEqual(response, {
'@olds/bad_example': {
'error': {
'name': 'NOT_FOUND',
'name': 'not_found',
'text': 'Could not find claim at "@olds/bad_example".',
}
}
@@ -2,7 +2,7 @@ import logging
import pathlib
import time
from ..blockchain.test_claim_commands import ClaimTestCase
from .test_claim_commands import ClaimTestCase
from lbry.conf import TranscodeConfig
from lbry.file_analysis import VideoFileAnalyzer
@@ -2,10 +2,8 @@ import asyncio
import lbry
import lbry.wallet
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
from lbry.wallet.network import ClientSession
from lbry.testcase import IntegrationTestCase, CommandTestCase
from lbry.wallet.orchstr8.node import SPVNode
from lbry.testcase import IntegrationTestCase
class TestSessions(IntegrationTestCase):
@@ -51,46 +49,3 @@ class TestSegwitServer(IntegrationTestCase):
async def test_at_least_it_starts(self):
await asyncio.wait_for(self.ledger.network.get_headers(0, 1), 1.0)
class TestUsagePayment(CommandTestCase):
async def test_single_server_payment(self):
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
wallet_pay_service.payment_period = 1
# only starts with a positive max key fee
wallet_pay_service.max_fee = "0.0"
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
self.assertFalse(wallet_pay_service.running)
wallet_pay_service.max_fee = "1.0"
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
self.assertTrue(wallet_pay_service.running)
await wallet_pay_service.stop()
await wallet_pay_service.start(ledger=self.ledger, wallet=self.wallet)
address = await self.blockchain.get_raw_change_address()
_, history = await self.ledger.get_local_status_and_history(address)
self.assertEqual(history, [])
node = SPVNode(self.conductor.spv_module, node_number=2)
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.1"})
self.addCleanup(node.stop)
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
await self.daemon.jsonrpc_wallet_reconnect()
features = await self.ledger.network.get_server_features()
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.1")
with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError):
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
await node.stop(False)
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.0"})
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
await self.daemon.jsonrpc_wallet_reconnect()
features = await self.ledger.network.get_server_features()
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.0")
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=3)
self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted
self.assertEqual(tx.outputs[0].amount, 100000000)
self.assertEqual(tx.outputs[0].get_address(self.ledger), address)
+2 -2
View File
@@ -88,7 +88,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
self.assertEqual(
response['lbry://@spam/hovercraft'],
{'error': {
'name': 'NOT_FOUND',
'name': 'not_found',
'text': 'Could not find claim at "lbry://@spam/hovercraft".'
}}
)
@@ -192,7 +192,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
self.assertEqual(
response[uri],
{'error': {
'name': 'NOT_FOUND',
'name': 'not_found',
'text': f'Could not find claim at "{uri}".'
}}
)
+2 -2
View File
@@ -7,7 +7,7 @@ from lbry.extras import cli
from lbry.extras.daemon.components import (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
)
from lbry.extras.daemon.daemon import Daemon
@@ -22,7 +22,7 @@ class CLIIntegrationTest(AsyncioTestCase):
conf.components_to_skip = (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
)
Daemon.component_attributes = {}
self.daemon = Daemon(conf)
@@ -365,7 +365,7 @@ class CommentCommands(CommandTestCase):
async def test04_comment_abandons(self):
rswanson = (await self.channel_create('@RonSwanson'))['outputs'][0]
stream = (await self.stream_create('Pawnee_Tow_Hall_of_Fame_by_Leslie_Knope'))['outputs'][0]
stream = (await self.stream_create('Pawnee Town Hall of Fame by Leslie Knope'))['outputs'][0]
comment = await self.daemon.jsonrpc_comment_create(
comment='KNOPE! WHAT DID I TELL YOU ABOUT PUTTING MY INFORMATION UP LIKE THAT',
claim_id=stream['claim_id'],
@@ -383,7 +383,7 @@ class CommentCommands(CommandTestCase):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
stream = await self.stream_create('Cool Lamps to Sit On', channel_id=moth_id)
claim_id = stream['outputs'][0]['claim_id']
comment1 = await self.daemon.jsonrpc_comment_create(
@@ -440,7 +440,7 @@ class CommentCommands(CommandTestCase):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
stream = await self.stream_create('Cool Lamps to Sit On', channel_id=moth_id)
claim_id = stream['outputs'][0]['claim_id']
hidden_comment = await self.daemon.jsonrpc_comment_create(
comment='Who on earth would want to sit around on a lamp all day',
+2 -17
View File
@@ -1,4 +1,3 @@
from lbry.extras.daemon.loggly_handler import get_loggly_handler
from lbry.testcase import CommandTestCase
@@ -12,7 +11,6 @@ class AddressManagement(CommandTestCase):
self.assertItemCount(single, 1)
self.assertEqual(single['items'][0], addresses['items'][11])
class SettingsManagement(CommandTestCase):
async def test_settings(self):
@@ -23,18 +21,5 @@ class SettingsManagement(CommandTestCase):
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('server', 50001))
setting = self.daemon.jsonrpc_settings_clear('lbryum_servers')
self.assertEqual(setting['lbryum_servers'][0], ('spv11.lbry.com', 50001))
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv11.lbry.com', 50001))
# test_privacy_settings (merged for reducing test time, unmerge when its fast)
# tests that changing share_usage_data propagates to the relevant properties
self.assertFalse(self.daemon.jsonrpc_settings_get()['share_usage_data'])
loggly = get_loggly_handler(self.daemon.conf)
self.addCleanup(loggly.close)
self.assertFalse(self.daemon.analytics_manager.enabled)
self.assertFalse(loggly.enabled)
self.daemon.jsonrpc_settings_set('share_usage_data', True)
self.assertTrue(self.daemon.jsonrpc_settings_get()['share_usage_data'])
self.assertTrue(self.daemon.analytics_manager.enabled)
self.assertTrue(loggly.enabled)
self.daemon.jsonrpc_settings_set('share_usage_data', False)
self.assertEqual(setting['lbryum_servers'][0], ('spv1.lbry.com', 50001))
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv1.lbry.com', 50001))
@@ -27,7 +27,6 @@ class TestComponentManager(AsyncioTestCase):
components.HashAnnouncerComponent,
components.PeerProtocolServerComponent,
components.StreamManagerComponent,
components.WalletServerPaymentsComponent
]
]
self.component_manager = ComponentManager(Config())
+4 -4
View File
@@ -57,6 +57,10 @@ class TestURLParsing(unittest.TestCase):
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
# unicode regex edges
_url = lambda name: url(name, stream_name=name)
_url('\u0009')
_url('\u000a')
_url('\u000d')
_url('\u0020')
_url('\uD799')
_url('\uE000')
_url('\uFFFD')
@@ -75,10 +79,6 @@ class TestURLParsing(unittest.TestCase):
fail("lbry://\uDFFE")
fail("lbry://\uFFFF")
fail("lbry://;")
fail("lbry://no\ttab")
fail("lbry://no space")
fail("lbry://no\rcr")
fail("lbry://no\new\nline")
fail("lbry://\"")
fail("lbry://\\")
fail("lbry:///")
-4
View File
@@ -122,10 +122,6 @@ async def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None):
class TestStreamManager(BlobExchangeTestBase):
async def asyncSetUp(self):
await super().asyncSetUp()
self.client_config.share_usage_data = True
async def setup_stream_manager(self, balance=10.0, fee=None, old_sort=False):
file_path = os.path.join(self.server_dir, "test_file")
with open(file_path, 'wb') as f:
+6 -12
View File
@@ -64,9 +64,8 @@ class CLILoggingTest(AsyncioTestCase):
self.assertTrue(log.isEnabledFor(logging.INFO))
self.assertFalse(log.isEnabledFor(logging.DEBUG))
self.assertFalse(log.isEnabledFor(logging.DEBUG))
self.assertEqual(len(log.handlers), 2)
self.assertEqual(len(log.handlers), 1)
self.assertIsInstance(log.handlers[0], logging.handlers.RotatingFileHandler)
self.assertFalse(log.handlers[1].enabled)
async with get_logger(["start", "--verbose"]) as log:
self.assertTrue(log.getChild("lbry").isEnabledFor(logging.DEBUG))
@@ -81,22 +80,16 @@ class CLILoggingTest(AsyncioTestCase):
self.assertFalse(log.getChild("torba").isEnabledFor(logging.DEBUG))
async def test_loggly(self):
async with get_logger(["start"]) as log: # default share_usage_data=False
log = log.getChild("lbry")
self.assertIsInstance(log.handlers[0], logging.StreamHandler)
self.assertIsInstance(log.handlers[1], logging.StreamHandler)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
self.assertFalse(log.handlers[2].enabled)
async with get_logger(["start"], share_usage_data=True) as log:
async with get_logger(["start"], share_usage_data=False) as log:
self.assertEqual(len(log.getChild("lbry").handlers), 2) # file and console
async with get_logger(["start"]) as log: # default share_usage_data=True
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 3)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
self.assertTrue(log.handlers[2].enabled)
async with get_logger(["start"], share_usage_data=False) as log: # explicit share_usage_data=False
async with get_logger(["start"], share_usage_data=True) as log: # explicit share_usage_data=True
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 3)
self.assertIsInstance(log.handlers[2], HTTPSLogglyHandler)
self.assertFalse(log.handlers[2].enabled)
async def test_quiet(self):
async with get_logger(["start"]) as log: # default is loud
@@ -107,6 +100,7 @@ class CLILoggingTest(AsyncioTestCase):
log = log.getChild("lbry")
self.assertEqual(len(log.handlers), 2)
self.assertIsNot(type(log.handlers[0]), logging.StreamHandler)
self.assertIsNot(type(log.handlers[1]), logging.StreamHandler)
class CLITest(AsyncioTestCase):
-12
View File
@@ -98,12 +98,6 @@ class TestQueryBuilder(unittest.TestCase):
'txo_name__in0_1': 'def456'
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.name__in': {'abc123'}}),
('txo.name = :txo_name__in0', {
'txo_name__in0': 'abc123',
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.age__in': 'SELECT age from ages_table'}),
('txo.age IN (SELECT age from ages_table)', {})
@@ -124,12 +118,6 @@ class TestQueryBuilder(unittest.TestCase):
'txo_name__not_in0_1': 'def456'
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.name__not_in': ('abc123',)}),
('txo.name != :txo_name__not_in0', {
'txo_name__not_in0': 'abc123',
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.age__not_in': 'SELECT age from ages_table'}),
('txo.age NOT IN (SELECT age from ages_table)', {})