Compare commits

..
Author SHA1 Message Date
Brannon King a58ebcbc53 triggers never activated 2020-02-05 13:18:15 -07:00
36 changed files with 281 additions and 427 deletions

No files matched your search

-79
View File
@@ -1,79 +0,0 @@
name: ci
on: pull_request
jobs:
lint:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- run: make install tools
- run: make lint
tests-unit:
name: "tests / unit"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- run: make install tools
- working-directory: lbry
env:
HOME: /tmp
run: coverage run -p --source=lbry -m unittest discover -vv tests.unit
tests-integration:
name: "tests / integration"
runs-on: ubuntu-latest
strategy:
matrix:
test:
- datanetwork
- blockchain
- other
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- if: matrix.test == 'other'
run: sudo apt install -y --no-install-recommends ffmpeg
- run: pip install tox-travis
- run: tox -e ${{ matrix.test }}
build:
needs: ["lint", "tests-unit", "tests-integration"]
name: "build"
strategy:
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-latest
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v1
with:
python-version: '3.7'
- name: Setup
run: |
pip install pyinstaller
pip install -e .
- if: startsWith(matrix.os, 'windows') == false
name: Build & Run (Unix)
run: |
pyinstaller --onefile --name lbrynet lbry/extras/cli.py
chmod +x dist/lbrynet
dist/lbrynet --version
- if: startsWith(matrix.os, 'windows')
name: Build & Run (Windows)
run: |
pip install pywin32
pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py
dist/lbrynet.exe --version
+4 -5
View File
@@ -46,14 +46,14 @@ test:datanetwork-integration:
test:blockchain-integration:
stage: test
script:
- apt-get update
- apt-get install -y --no-install-recommends ffmpeg
- pip install tox-travis
- tox -e blockchain
test:other-integration:
stage: test
script:
- apt-get update
- apt-get install -y --no-install-recommends ffmpeg
- pip install tox-travis
- tox -e other
@@ -84,15 +84,14 @@ test:json-api:
build:linux:
extends: .build
image: ubuntu:16.04
image: ubuntu:18.04 # cant use python3.7 image - binary won't run on ubuntu
variables:
OS: linux
before_script:
- apt-get update
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
- add-apt-repository -y ppa:deadsnakes/ppa
- apt-get update
- apt-get install -y --no-install-recommends python3.7-dev
- apt-get install -y --no-install-recommends python3.7-dev python3-setuptools python3-wheel
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
build:mac:
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.59.1"
__version__ = "0.58.1"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+9 -14
View File
@@ -573,15 +573,15 @@ class Config(CLIConfig):
('reflector.lbry.com', 5566)
])
lbryum_servers = Servers("SPV wallet servers", [
('spv11.lbry.com', 50001),
('spv12.lbry.com', 50001),
('spv13.lbry.com', 50001),
('spv14.lbry.com', 50001),
('spv15.lbry.com', 50001),
('spv16.lbry.com', 50001),
('spv17.lbry.com', 50001),
('spv18.lbry.com', 50001),
('spv19.lbry.com', 50001),
('spv1.lbry.com', 50001),
('spv2.lbry.com', 50001),
('spv3.lbry.com', 50001),
('spv4.lbry.com', 50001),
('spv5.lbry.com', 50001),
('spv6.lbry.com', 50001),
('spv7.lbry.com', 50001),
('spv8.lbry.com', 50001),
('spv9.lbry.com', 50001),
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.com', 4444), # US EAST
@@ -616,11 +616,6 @@ class Config(CLIConfig):
"Strategy to use when selecting UTXOs for a transaction",
STRATEGIES, "standard")
save_resolved_claims = Toggle(
"Save content claims to the database when they are resolved to keep file_list up to date, "
"only disable this if file_x commands are not needed", True
)
@property
def streaming_host(self):
return self.streaming_server.split(':')[0]
+1 -1
View File
@@ -53,7 +53,7 @@ Code | Name | Message
407 | DataDownload | Failed to download blob. *generic*
410 | Resolve | Failed to resolve '{url}'.
411 | ResolveTimeout | Failed to resolve '{url}' within the timeout.
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.
411 | ResolveCensored | Resolve of '{url}' was censored by channel with claim id '{censor_id}'.
420 | KeyFeeAboveMaxAllowed | {message}
421 | InvalidPassword | Password is invalid.
422 | IncompatibleWalletServer | '{server}:{port}' has an incompatibly old version.
+3 -35
View File
@@ -1,4 +1,4 @@
from .base import BaseError, claim_id
from .base import BaseError
class UserInputError(BaseError):
@@ -16,22 +16,18 @@ class CommandError(UserInputError):
class CommandDoesNotExistError(CommandError):
def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' does not exist.")
class CommandDeprecatedError(CommandError):
def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' is deprecated.")
class CommandInvalidArgumentError(CommandError):
def __init__(self, argument, command):
self.argument = argument
self.command = command
super().__init__(f"Invalid argument '{argument}' to command '{command}'.")
@@ -41,7 +37,6 @@ class CommandTemporarilyUnavailableError(CommandError):
"""
def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' is temporarily unavailable.")
@@ -51,7 +46,6 @@ class CommandPermanentlyUnavailableError(CommandError):
"""
def __init__(self, command):
self.command = command
super().__init__(f"Command '{command}' is permanently unavailable.")
@@ -64,15 +58,12 @@ class InputValueError(UserInputError, ValueError):
class GenericInputValueError(InputValueError):
def __init__(self, value, argument):
self.value = value
self.argument = argument
super().__init__(f"The value '{value}' for argument '{argument}' is not valid.")
class InputValueIsNoneError(InputValueError):
def __init__(self, argument):
self.argument = argument
super().__init__(f"None or null is not valid value for argument '{argument}'.")
@@ -88,7 +79,6 @@ class ConfigWriteError(ConfigurationError):
"""
def __init__(self, path):
self.path = path
super().__init__(f"Cannot write configuration file '{path}'.")
@@ -98,7 +88,6 @@ class ConfigReadError(ConfigurationError):
"""
def __init__(self, path):
self.path = path
super().__init__(f"Cannot find provided configuration file '{path}'.")
@@ -108,21 +97,18 @@ class ConfigParseError(ConfigurationError):
"""
def __init__(self, path):
self.path = path
super().__init__(f"Failed to parse the configuration file '{path}'.")
class ConfigMissingError(ConfigurationError):
def __init__(self, path):
self.path = path
super().__init__(f"Configuration file '{path}' is missing setting that has no default / fallback.")
class ConfigInvalidError(ConfigurationError):
def __init__(self, path):
self.path = path
super().__init__(f"Configuration file '{path}' has setting with invalid value.")
@@ -202,29 +188,24 @@ class DataDownloadError(WalletError):
class ResolveError(WalletError):
def __init__(self, url):
self.url = url
super().__init__(f"Failed to resolve '{url}'.")
class ResolveTimeoutError(WalletError):
def __init__(self, url):
self.url = url
super().__init__(f"Failed to resolve '{url}' within the timeout.")
class ResolveCensoredError(WalletError):
def __init__(self, url, censor_hash):
self.url = url
self.censor_hash = censor_hash
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{claim_id(censor_hash)}'.")
def __init__(self, url, censor_id):
super().__init__(f"Resolve of '{url}' was censored by channel with claim id '{censor_id}'.")
class KeyFeeAboveMaxAllowedError(WalletError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
@@ -237,8 +218,6 @@ class InvalidPasswordError(WalletError):
class IncompatibleWalletServerError(WalletError):
def __init__(self, server, port):
self.server = server
self.port = port
super().__init__(f"'{server}:{port}' has an incompatibly old version.")
@@ -299,35 +278,30 @@ class DownloadCancelledError(BlobError):
class DownloadSDTimeoutError(BlobError):
def __init__(self, download):
self.download = download
super().__init__(f"Failed to download sd blob {download} within timeout.")
class DownloadDataTimeoutError(BlobError):
def __init__(self, download):
self.download = download
super().__init__(f"Failed to download data blobs for sd hash {download} within timeout.")
class InvalidStreamDescriptorError(BlobError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
class InvalidDataError(BlobError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
class InvalidBlobHashError(BlobError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
@@ -340,14 +314,12 @@ class ComponentError(BaseError):
class ComponentStartConditionNotMetError(ComponentError):
def __init__(self, components):
self.components = components
super().__init__(f"Unresolved dependencies for: {components}")
class ComponentsNotStartedError(ComponentError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
@@ -360,20 +332,16 @@ class CurrencyExchangeError(BaseError):
class InvalidExchangeRateResponseError(CurrencyExchangeError):
def __init__(self, source, reason):
self.source = source
self.reason = reason
super().__init__(f"Failed to get exchange rate from {source}: {reason}")
class CurrencyConversionError(CurrencyExchangeError):
def __init__(self, message):
self.message = message
super().__init__(f"{message}")
class InvalidCurrencyError(CurrencyExchangeError):
def __init__(self, currency):
self.currency = currency
super().__init__(f"Invalid currency: {currency} is not a supported currency.")
-7
View File
@@ -1,9 +1,2 @@
from binascii import hexlify
def claim_id(claim_hash):
return hexlify(claim_hash[::-1]).decode()
class BaseError(Exception):
pass
+4 -17
View File
@@ -13,12 +13,10 @@ class {name}({parents}):{doc}
"""
INIT = """
def __init__({args}):{fields}
def __init__({args}):
super().__init__({format}"{message}")
"""
FUNCTIONS = ['claim_id']
class ErrorClass:
@@ -52,20 +50,10 @@ class ErrorClass:
def get_arguments(self):
args = ['self']
for arg in re.findall('{([a-z0-1_()]+)}', self.message):
for func in FUNCTIONS:
if arg.startswith(f'{func}('):
arg = arg[len(f'{func}('):-1]
break
for arg in re.findall('{([a-z0-1_]+)}', self.message):
args.append(arg)
return args
@staticmethod
def get_fields(args):
if len(args) > 1:
return f''.join(f'\n{INDENT*2}self.{field} = {field}' for field in args[1:])
return ''
@staticmethod
def get_doc_string(doc):
if doc:
@@ -81,8 +69,7 @@ class ErrorClass:
args = self.get_arguments()
if self.is_leaf:
out.write((CLASS + INIT).format(
name=self.class_name, parents=', '.join(parents),
args=', '.join(args), fields=self.get_fields(args),
name=self.class_name, parents=', '.join(parents), args=', '.join(args),
message=self.message, doc=self.get_doc_string(self.comment), format='f' if len(args) > 1 else ''
))
else:
@@ -115,7 +102,7 @@ def find_parent(stack, child):
def generate(out):
out.write(f"from .base import BaseError, {', '.join(FUNCTIONS)}\n")
out.write('from .base import BaseError\n')
stack = {}
for error in get_errors():
error.render(out, find_parent(stack, error))
+18 -49
View File
@@ -297,7 +297,6 @@ class Daemon(metaclass=JSONRPCServerType):
def __init__(self, conf: Config, component_manager: typing.Optional[ComponentManager] = None):
self.conf = conf
self.platform_info = system_info.get_platform()
self._video_file_analyzer = VideoFileAnalyzer(conf)
self._node_id = None
self._installation_id = None
@@ -308,7 +307,7 @@ class Daemon(metaclass=JSONRPCServerType):
skip_components=conf.components_to_skip or []
)
self.component_startup_task = None
self._connection_status: typing.Tuple[float, bool] = (self.component_manager.loop.time(), False)
self._connection_status: typing.Tuple[float, bool] = [self.component_manager.loop.time(), False]
logging.getLogger('aiohttp.access').setLevel(logging.WARN)
rpc_app = web.Application()
@@ -450,7 +449,7 @@ class Daemon(metaclass=JSONRPCServerType):
async def start(self):
log.info("Starting LBRYNet Daemon")
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
log.info("Platform: %s", json.dumps(self.platform_info, indent=2))
log.info("Platform: %s", json.dumps(system_info.get_platform(), indent=2))
await self.analytics_manager.send_server_startup()
await self.rpc_runner.setup()
await self.streaming_runner.setup()
@@ -895,7 +894,9 @@ class Daemon(metaclass=JSONRPCServerType):
'build': (str) "dev" | "qa" | "rc" | "release",
}
"""
return self.platform_info
platform_info = system_info.get_platform()
log.info("Get version info: %s", json.dumps(platform_info))
return platform_info
@requires(WALLET_COMPONENT)
async def jsonrpc_resolve(self, urls: typing.Union[str, list], wallet_id=None):
@@ -2153,7 +2154,7 @@ class Daemon(metaclass=JSONRPCServerType):
List my stream and channel claims.
Usage:
claim_list [--claim_type=<claim_type>...]
claim_list [--claim_type=<claim_type>]
[--account_id=<account_id>] [--wallet_id=<wallet_id>]
[--page=<page>] [--page_size=<page_size>]
[--resolve]
@@ -2179,12 +2180,7 @@ class Daemon(metaclass=JSONRPCServerType):
return paginate_rows(claims, claim_count, page, page_size, claim_type=claim_type, resolve=resolve)
@requires(WALLET_COMPONENT)
async def jsonrpc_claim_search(self, claim_ids=None, channel_ids=None, not_channel_ids=None,
has_channel_signature=False, valid_channel_signature=False,
invalid_channel_signature=False, is_controlling=False, stream_types=None,
media_types=None, any_tags=None, all_tags=None, not_tags=None, any_languages=None,
all_languages=None, not_languages=None, any_locations=None, all_locations=None,
not_locations=None, order_by=None, **kwargs):
async def jsonrpc_claim_search(self, **kwargs):
"""
Search for stream and channel claims on the blockchain.
@@ -2321,50 +2317,23 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
_kwargs = {}
_kwargs.update(kwargs)
_kwargs.update(
{
'claim_ids': claim_ids or [],
'channel_ids': channel_ids or [],
'not_channel_ids': not_channel_ids or [],
'has_channel_signature': has_channel_signature,
'valid_channel_signature': valid_channel_signature,
'invalid_channel_signature': invalid_channel_signature,
'is_controlling': is_controlling,
'stream_types': stream_types or [],
'media_types': media_types or [],
'any_tags': any_tags or [],
'all_tags': all_tags or [],
'not_tags': not_tags or [],
'any_languages': any_languages or [],
'all_languages': all_languages or [],
'not_languages': not_languages or [],
'any_locations': any_locations or [],
'all_locations': all_locations or [],
'not_locations': not_locations or [],
'order_by': order_by or []
}
)
wallet = self.wallet_manager.get_wallet_or_default(_kwargs.pop('wallet_id', None))
if {'claim_id', 'claim_ids'}.issubset(_kwargs):
wallet = self.wallet_manager.get_wallet_or_default(kwargs.pop('wallet_id', None))
if {'claim_id', 'claim_ids'}.issubset(kwargs):
raise ValueError("Only 'claim_id' or 'claim_ids' is allowed, not both.")
if _kwargs.pop('valid_channel_signature', False):
_kwargs['signature_valid'] = 1
if _kwargs.pop('invalid_channel_signature', False):
_kwargs['signature_valid'] = 0
page_num, page_size = abs(_kwargs.pop('page', 1)), min(abs(_kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
_kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
txos, blocked, _, total = await self.ledger.claim_search(wallet.accounts, **_kwargs)
if kwargs.pop('valid_channel_signature', False):
kwargs['signature_valid'] = 1
if kwargs.pop('invalid_channel_signature', False):
kwargs['signature_valid'] = 0
page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
txos, blocked, _, total = await self.ledger.claim_search(wallet.accounts, **kwargs)
result = {
"items": txos,
"blocked": blocked,
"page": page_num,
"page_size": page_size
}
if not _kwargs.pop('no_totals', False):
if not kwargs.pop('no_totals', False):
result['total_pages'] = int((total + (page_size - 1)) / page_size)
result['total_items'] = total
return result
@@ -4884,7 +4853,7 @@ class Daemon(metaclass=JSONRPCServerType):
async def resolve(self, accounts, urls):
results = await self.ledger.resolve(accounts, urls)
if self.conf.save_resolved_claims and results:
if results:
try:
claims = self.stream_manager._convert_to_old_resolve_output(self.wallet_manager, results)
await self.storage.save_claims_for_resolve([
+5 -4
View File
@@ -25,7 +25,7 @@ class VideoFileAnalyzer:
process = await asyncio.create_subprocess_exec(self._conf.ffmpeg_folder + command, *args,
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
stdout, stderr = await process.communicate() # returns when the streams are closed
return stdout.decode(errors='replace') + stderr.decode(errors='replace'), process.returncode
return stdout.decode() + stderr.decode(), process.returncode
async def _verify_executable(self, name):
try:
@@ -119,9 +119,10 @@ class VideoFileAnalyzer:
return ""
result, _ = await self._execute("ffprobe", f'-v debug "{video_file}"')
match = re.search(r"Before avformat_find_stream_info.+?\s+seeks:(\d+)\s+", result)
if match and int(match.group(1)) != 0:
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
iterator = re.finditer(r"\s+seeks:(\d+)\s+", result)
for match in iterator:
if int(match.group(1)) != 0:
return "Video stream descriptors are not at the start of the file (the faststart flag was not used)."
return ""
@staticmethod
+31 -36
View File
@@ -13,16 +13,6 @@ NOT_FOUND = ErrorMessage.Code.Name(ErrorMessage.NOT_FOUND)
BLOCKED = ErrorMessage.Code.Name(ErrorMessage.BLOCKED)
def set_reference(reference, claim_hash, rows):
if claim_hash:
for txo in rows:
if claim_hash == txo['claim_hash']:
reference.tx_hash = txo['txo_hash'][:32]
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
reference.height = txo['height']
return
class Censor:
__slots__ = 'streams', 'channels', 'censored', 'total'
@@ -49,12 +39,12 @@ class Censor:
self.total += 1
return was_censored
def to_message(self, outputs: OutputsMessage, extra_txo_rows):
def to_message(self, outputs: OutputsMessage):
outputs.blocked_total = self.total
for censoring_channel_hash, count in self.censored.items():
blocked = outputs.blocked.add()
blocked.count = count
set_reference(blocked.channel, censoring_channel_hash, extra_txo_rows)
block = outputs.blocked.add()
block.count = count
block.channel_hash = censoring_channel_hash
class Outputs:
@@ -76,35 +66,26 @@ class Outputs:
for txo_message in self.extra_txos:
self.message_to_txo(txo_message, tx_map)
txos = [self.message_to_txo(txo_message, tx_map) for txo_message in self.txos]
return txos, self.inflate_blocked(tx_map)
return txos, self.inflate_blocked()
def inflate_blocked(self, tx_map):
def inflate_blocked(self):
return {
"total": self.blocked_total,
"channels": [{
'channel': self.message_to_txo(blocked.channel, tx_map),
'blocked': blocked.count
} for blocked in self.blocked]
"channels": {
hexlify(message.channel_hash[::-1]).decode(): message.count
for message in self.blocked
}
}
def message_to_txo(self, txo_message, tx_map):
if txo_message.WhichOneof('meta') == 'error':
error = {
return {
'error': {
'name': txo_message.error.Code.Name(txo_message.error.code),
'name': txo_message.error.Code.Name(txo_message.error.code).lower(),
'text': txo_message.error.text,
}
}
if error['error']['name'] == BLOCKED:
error['error']['censor'] = self.message_to_txo(
txo_message.error.blocked.channel, tx_map
)
return error
tx = tx_map.get(txo_message.tx_hash)
if not tx:
return
txo = tx.outputs[txo_message.nout]
txo = tx_map[txo_message.tx_hash].outputs[txo_message.nout]
if txo_message.WhichOneof('meta') == 'claim':
claim = txo_message.claim
txo.meta = {
@@ -164,7 +145,7 @@ class Outputs:
if total is not None:
page.total = total
if blocked is not None:
blocked.to_message(page, extra_txo_rows)
blocked.to_message(page)
for row in txo_rows:
cls.row_to_message(row, page.txos.add(), extra_txo_rows)
for row in extra_txo_rows:
@@ -181,7 +162,6 @@ class Outputs:
txo_message.error.code = ErrorMessage.NOT_FOUND
elif isinstance(txo, ResolveCensoredError):
txo_message.error.code = ErrorMessage.BLOCKED
set_reference(txo_message.error.blocked.channel, txo.censor_hash, extra_txo_rows)
return
txo_message.tx_hash = txo['txo_hash'][:32]
txo_message.nout, = struct.unpack('<I', txo['txo_hash'][32:])
@@ -204,5 +184,20 @@ class Outputs:
txo_message.claim.trending_mixed = txo['trending_mixed']
txo_message.claim.trending_local = txo['trending_local']
txo_message.claim.trending_global = txo['trending_global']
set_reference(txo_message.claim.channel, txo['channel_hash'], extra_txo_rows)
set_reference(txo_message.claim.repost, txo['reposted_claim_hash'], extra_txo_rows)
cls.set_reference(txo_message, 'channel', txo['channel_hash'], extra_txo_rows)
cls.set_reference(txo_message, 'repost', txo['reposted_claim_hash'], extra_txo_rows)
@staticmethod
def set_blocked(message, blocked):
message.blocked_total = blocked.total
@staticmethod
def set_reference(message, attr, claim_hash, rows):
if claim_hash:
for txo in rows:
if claim_hash == txo['claim_hash']:
reference = getattr(message.claim, attr)
reference.tx_hash = txo['txo_hash'][:32]
reference.nout = struct.unpack('<I', txo['txo_hash'][32:])[0]
reference.height = txo['height']
break
+5 -6
View File
@@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
name='result.proto',
package='pb',
syntax='proto3',
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\"5\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x1b\n\x07\x63hannel\x18\x02 \x01(\x0b\x32\n.pb.Outputb\x06proto3')
serialized_pb=_b('\n\x0cresult.proto\x12\x02pb\"\x97\x01\n\x07Outputs\x12\x18\n\x04txos\x18\x01 \x03(\x0b\x32\n.pb.Output\x12\x1e\n\nextra_txos\x18\x02 \x03(\x0b\x32\n.pb.Output\x12\r\n\x05total\x18\x03 \x01(\r\x12\x0e\n\x06offset\x18\x04 \x01(\r\x12\x1c\n\x07\x62locked\x18\x05 \x03(\x0b\x32\x0b.pb.Blocked\x12\x15\n\rblocked_total\x18\x06 \x01(\r\"{\n\x06Output\x12\x0f\n\x07tx_hash\x18\x01 \x01(\x0c\x12\x0c\n\x04nout\x18\x02 \x01(\r\x12\x0e\n\x06height\x18\x03 \x01(\r\x12\x1e\n\x05\x63laim\x18\x07 \x01(\x0b\x32\r.pb.ClaimMetaH\x00\x12\x1a\n\x05\x65rror\x18\x0f \x01(\x0b\x32\t.pb.ErrorH\x00\x42\x06\n\x04meta\"\xaf\x03\n\tClaimMeta\x12\x1b\n\x07\x63hannel\x18\x01 \x01(\x0b\x32\n.pb.Output\x12\x1a\n\x06repost\x18\x02 \x01(\x0b\x32\n.pb.Output\x12\x11\n\tshort_url\x18\x03 \x01(\t\x12\x15\n\rcanonical_url\x18\x04 \x01(\t\x12\x16\n\x0eis_controlling\x18\x05 \x01(\x08\x12\x18\n\x10take_over_height\x18\x06 \x01(\r\x12\x17\n\x0f\x63reation_height\x18\x07 \x01(\r\x12\x19\n\x11\x61\x63tivation_height\x18\x08 \x01(\r\x12\x19\n\x11\x65xpiration_height\x18\t \x01(\r\x12\x19\n\x11\x63laims_in_channel\x18\n \x01(\r\x12\x10\n\x08reposted\x18\x0b \x01(\r\x12\x18\n\x10\x65\x66\x66\x65\x63tive_amount\x18\x14 \x01(\x04\x12\x16\n\x0esupport_amount\x18\x15 \x01(\x04\x12\x16\n\x0etrending_group\x18\x16 \x01(\r\x12\x16\n\x0etrending_mixed\x18\x17 \x01(\x02\x12\x16\n\x0etrending_local\x18\x18 \x01(\x02\x12\x17\n\x0ftrending_global\x18\x19 \x01(\x02\"\x94\x01\n\x05\x45rror\x12\x1c\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x0e.pb.Error.Code\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x1c\n\x07\x62locked\x18\x03 \x01(\x0b\x32\x0b.pb.Blocked\"A\n\x04\x43ode\x12\x10\n\x0cUNKNOWN_CODE\x10\x00\x12\r\n\tNOT_FOUND\x10\x01\x12\x0b\n\x07INVALID\x10\x02\x12\x0b\n\x07\x42LOCKED\x10\x03\".\n\x07\x42locked\x12\r\n\x05\x63ount\x18\x01 \x01(\r\x12\x14\n\x0c\x63hannel_hash\x18\x02 \x01(\x0c\x62\x06proto3')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -388,9 +388,9 @@ _BLOCKED = _descriptor.Descriptor(
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='channel', full_name='pb.Blocked.channel', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
name='channel_hash', full_name='pb.Blocked.channel_hash', index=1,
number=2, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
@@ -407,7 +407,7 @@ _BLOCKED = _descriptor.Descriptor(
oneofs=[
],
serialized_start=884,
serialized_end=937,
serialized_end=930,
)
_OUTPUTS.fields_by_name['txos'].message_type = _OUTPUT
@@ -426,7 +426,6 @@ _CLAIMMETA.fields_by_name['repost'].message_type = _OUTPUT
_ERROR.fields_by_name['code'].enum_type = _ERROR_CODE
_ERROR.fields_by_name['blocked'].message_type = _BLOCKED
_ERROR_CODE.containing_type = _ERROR
_BLOCKED.fields_by_name['channel'].message_type = _OUTPUT
DESCRIPTOR.message_types_by_name['Outputs'] = _OUTPUTS
DESCRIPTOR.message_types_by_name['Output'] = _OUTPUT
DESCRIPTOR.message_types_by_name['ClaimMeta'] = _CLAIMMETA
+1 -1
View File
@@ -7,7 +7,7 @@ def _create_url_regex():
# see https://spec.lbry.com/ and test_url.py
invalid_names_regex = \
r"[^=&#:$@%?;\"/\\<>%{}|^~`\[\]" \
r"\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+"
r"\u0000-\u0008\u000b-\u000c\u000e-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]+"
def _named(name, regex):
return "(?P<" + name + ">" + regex + ")"
+16 -16
View File
@@ -1,6 +1,7 @@
import logging
import asyncio
import sqlite3
import sys
from binascii import hexlify
from concurrent.futures.thread import ThreadPoolExecutor
@@ -49,6 +50,11 @@ class AIOSQLite:
def executescript(self, script: str) -> Awaitable:
return self.run(lambda conn: conn.executescript(script))
def create_function(self, name, num_params, func, deterministic=True) -> Awaitable:
if sys.version_info < (3, 8):
return self.run(lambda conn: conn.create_function(name, num_params, func))
return self.run(lambda conn: conn.create_function(name, num_params, func, deterministic=deterministic))
def execute_fetchall(self, sql: str, parameters: Iterable = None) -> Awaitable[Iterable[sqlite3.Row]]:
parameters = parameters if parameters is not None else []
return self.run(lambda conn: conn.execute(sql, parameters).fetchall())
@@ -134,20 +140,16 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
col, op = col[:-len('__not_like')], 'NOT LIKE'
elif key.endswith('__in') or key.endswith('__not_in'):
if key.endswith('__in'):
col, op, one_val_op = col[:-len('__in')], 'IN', '='
col, op = col[:-len('__in')], 'IN'
else:
col, op, one_val_op = col[:-len('__not_in')], 'NOT IN', '!='
col, op = col[:-len('__not_in')], 'NOT IN'
if constraint:
if isinstance(constraint, (list, set, tuple)):
if len(constraint) == 1:
values[f'{key}{tag}'] = next(iter(constraint))
sql.append(f'{col} {one_val_op} :{key}{tag}')
else:
keys = []
for i, val in enumerate(constraint):
keys.append(f':{key}{tag}_{i}')
values[f'{key}{tag}_{i}'] = val
sql.append(f'{col} {op} ({", ".join(keys)})')
keys = []
for i, val in enumerate(constraint):
keys.append(f':{key}{tag}_{i}')
values[f'{key}{tag}_{i}'] = val
sql.append(f'{col} {op} ({", ".join(keys)})')
elif isinstance(constraint, str):
sql.append(f'{col} {op} ({constraint})')
else:
@@ -743,11 +745,9 @@ class Database(SQLiteMixin):
@staticmethod
def constrain_claims(constraints):
claim_types = constraints.pop('claim_type', None)
if isinstance(claim_types, str) and claim_types:
claim_types = [claim_types]
if isinstance(claim_types, list) and claim_types:
constraints['txo_type__in'] = [TXO_TYPES[ct] for ct in claim_types]
claim_type = constraints.pop('claim_type', None)
if claim_type is not None:
constraints['txo_type'] = TXO_TYPES[claim_type]
else:
constraints['txo_type__in'] = CLAIM_TYPES
+1 -2
View File
@@ -630,8 +630,7 @@ class Ledger(metaclass=LedgerRegistry):
raise asyncio.TimeoutError('Timed out waiting for transaction.')
async def _inflate_outputs(self, query, accounts) -> Tuple[List[Output], dict, int, int]:
encoded_outputs = await query
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
outputs = Outputs.from_base64(await query)
txs = []
if len(outputs.txs) > 0:
txs: List[Transaction] = await asyncio.gather(*(
+1 -1
View File
@@ -148,7 +148,7 @@ class ClientSession(BaseClientSession):
class Network:
PROTOCOL_VERSION = __version__
MINIMUM_REQUIRED = (0, 59, 0)
MINIMUM_REQUIRED = (0, 57, 0)
def __init__(self, ledger):
self.ledger = ledger
-2
View File
@@ -228,7 +228,6 @@ class BlockchainProcess(asyncio.SubprocessProtocol):
def process_exited(self):
self.stopped.set()
self.ready.set()
class BlockchainNode:
@@ -316,7 +315,6 @@ class BlockchainNode:
BlockchainProcess, *command
)
await self.protocol.ready.wait()
assert not self.protocol.stopped.is_set()
async def stop(self, cleanup=True):
try:
+3 -3
View File
@@ -164,7 +164,6 @@ class BlockProcessor:
# is consistent with self.height
self.state_lock = asyncio.Lock()
self.search_cache = {}
async def run_in_thread_with_lock(self, func, *args):
# Run in a thread to prevent blocking. Shielded so that
# cancellations from shutdown don't lose work - when the task
@@ -192,8 +191,6 @@ class BlockProcessor:
if hprevs == chain:
start = time.perf_counter()
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
for cache in self.search_cache.values():
cache.clear()
await self._maybe_flush()
processed_time = time.perf_counter() - start
BLOCK_COUNT.set(self.height)
@@ -732,6 +729,7 @@ class LBRYBlockProcessor(BlockProcessor):
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
self.sql: SQLDB = self.db.sql
self.timer = Timer('BlockProcessor')
self.search_cache = {}
def advance_blocks(self, blocks):
self.sql.begin()
@@ -746,6 +744,8 @@ class LBRYBlockProcessor(BlockProcessor):
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
if self.env.individual_tag_indexes:
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
for cache in self.search_cache.values():
cache.clear()
def advance_txs(self, height, txs, header):
timer = self.timer.sub_timers['advance_blocks']
-1
View File
@@ -1,7 +1,6 @@
CLAIM_TYPES = {
'stream': 1,
'channel': 2,
'repost': 3
}
STREAM_TYPES = {
+17 -21
View File
@@ -4,11 +4,10 @@ import apsw
import logging
from operator import itemgetter
from typing import Tuple, List, Dict, Union, Type, Optional
from binascii import unhexlify
from binascii import unhexlify, hexlify
from decimal import Decimal
from contextvars import ContextVar
from functools import wraps
from itertools import chain
from dataclasses import dataclass
from lbry.wallet.database import query, interpolate
@@ -266,7 +265,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
else:
constraints['claim.claim_id__like'] = f'{claim_id[:40]}%'
elif 'claim_ids' in constraints:
constraints['claim.claim_id__in'] = set(constraints.pop('claim_ids'))
constraints['claim.claim_id__in'] = constraints.pop('claim_ids')
if 'reposted_claim_id' in constraints:
constraints['claim.reposted_claim_hash'] = unhexlify(constraints.pop('reposted_claim_id'))[::-1]
@@ -282,15 +281,15 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
if 'channel_ids' in constraints:
channel_ids = constraints.pop('channel_ids')
if channel_ids:
constraints['claim.channel_hash__in'] = {
constraints['claim.channel_hash__in'] = [
unhexlify(cid)[::-1] for cid in channel_ids
}
]
if 'not_channel_ids' in constraints:
not_channel_ids = constraints.pop('not_channel_ids')
if not_channel_ids:
not_channel_ids_binary = {
not_channel_ids_binary = [
unhexlify(ncid)[::-1] for ncid in not_channel_ids
}
]
if constraints.get('has_channel_signature', False):
constraints['claim.channel_hash__not_in'] = not_channel_ids_binary
else:
@@ -320,13 +319,13 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
if 'stream_types' in constraints:
stream_types = constraints.pop('stream_types')
if stream_types:
constraints['claim.stream_type__in'] = {
constraints['claim.stream_type__in'] = [
STREAM_TYPES[stream_type] for stream_type in stream_types
}
]
if 'media_types' in constraints:
media_types = constraints.pop('media_types')
if media_types:
constraints['claim.media_type__in'] = set(media_types)
constraints['claim.media_type__in'] = media_types
if 'fee_currency' in constraints:
constraints['claim.fee_currency'] = constraints.pop('fee_currency').lower()
@@ -389,13 +388,9 @@ def search_claims(censor: Censor, **constraints) -> List:
)
def _get_referenced_rows(txo_rows: List[dict], censor_channels: List[bytes]):
censor = ctx.get().get_resolve_censor()
def _get_referenced_rows(censor: Censor, txo_rows: List[dict]):
repost_hashes = set(filter(None, map(itemgetter('reposted_claim_hash'), txo_rows)))
channel_hashes = set(chain(
filter(None, map(itemgetter('channel_hash'), txo_rows)),
censor_channels
))
channel_hashes = set(filter(None, map(itemgetter('channel_hash'), txo_rows)))
reposted_txos = []
if repost_hashes:
@@ -418,10 +413,12 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
total = count_claims(**constraints)
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
if 'order_by' not in constraints:
constraints['order_by'] = ["claim_hash"]
context = ctx.get()
search_censor = context.get_search_censor()
txo_rows = search_claims(search_censor, **constraints)
extra_txo_rows = _get_referenced_rows(txo_rows, search_censor.censored.keys())
extra_txo_rows = _get_referenced_rows(context.get_resolve_censor(), txo_rows)
return txo_rows, extra_txo_rows, constraints['offset'], total, search_censor
@@ -429,8 +426,7 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
def resolve(urls) -> Tuple[List, List]:
txo_rows = [resolve_url(raw_url) for raw_url in urls]
extra_txo_rows = _get_referenced_rows(
[txo for txo in txo_rows if isinstance(txo, dict)],
[txo.censor_hash for txo in txo_rows if isinstance(txo, ResolveCensoredError)]
ctx.get().get_resolve_censor(), [r for r in txo_rows if isinstance(r, dict)]
)
return txo_rows, extra_txo_rows
@@ -456,7 +452,7 @@ def resolve_url(raw_url):
if matches:
channel = matches[0]
elif censor.censored:
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
return ResolveCensoredError(raw_url, hexlify(next(iter(censor.censored))[::-1]).decode())
else:
return LookupError(f'Could not find channel in "{raw_url}".')
@@ -476,7 +472,7 @@ def resolve_url(raw_url):
if matches:
return matches[0]
elif censor.censored:
return ResolveCensoredError(raw_url, next(iter(censor.censored)))
return ResolveCensoredError(raw_url, hexlify(next(iter(censor.censored))[::-1]).decode())
else:
return LookupError(f'Could not find claim at "{raw_url}".')
+22 -26
View File
@@ -24,23 +24,16 @@ DECAY_PER_RENORM = DECAY**(RENORM_INTERVAL)
TRENDING_LOG = True
# Stubs
def install(connection):
"""
Install the AR trending algorithm.
"""
check_trending_values(connection)
if TRENDING_LOG:
f = open("trending_ar.log", "a")
f.close()
# Stub
CREATE_TREND_TABLE = ""
def check_trending_values(connection):
"""
If the trending values appear to be based on the zscore algorithm,
If the trending values appear to be based on the standard algorithm,
reset them. This will allow resyncing from a standard snapshot.
"""
c = connection.cursor()
@@ -60,6 +53,9 @@ def check_trending_values(connection):
COMMIT;""")
print("done.")
# Create the index
c.execute("create index if not exists claim_id_only_idx on claim (claim_id);")
def spike_height(trending_score, x, x_old, time_boost=1.0):
"""
@@ -75,9 +71,8 @@ def spike_height(trending_score, x, x_old, time_boost=1.0):
# Softened change in amount counts more for minnows
if delta > 0.0:
if trending_score >= 0.0:
multiplier = 0.1/((trending_score/time_boost + softened_change_in_amount) + 1.0)
softened_change_in_amount *= multiplier
multiplier = 1.0/math.sqrt(x + 1.0)
softened_change_in_amount *= multiplier
else:
softened_change_in_amount *= -1.0
@@ -111,14 +106,14 @@ class TrendingData:
# Have all claims been read from db yet?
self.initialised = False
def insert_claim_from_load(self, claim_hash, trending_score, total_amount):
def insert_claim_from_load(self, claim_id, trending_score, total_amount):
assert not self.initialised
self.claims[claim_hash] = {"trending_score": trending_score,
"total_amount": total_amount,
"changed": False}
self.claims[claim_id] = {"trending_score": trending_score,
"total_amount": total_amount,
"changed": False}
def update_claim(self, claim_hash, total_amount, time_boost=1.0):
def update_claim(self, claim_id, total_amount, time_boost=1.0):
"""
Update trending data for a claim, given its new total amount.
"""
@@ -126,8 +121,8 @@ class TrendingData:
# Extract existing total amount and trending score
# or use starting values if the claim is new
if claim_hash in self.claims:
old_state = copy.deepcopy(self.claims[claim_hash])
if claim_id in self.claims:
old_state = copy.deepcopy(self.claims[claim_id])
else:
old_state = {"trending_score": 0.0,
"total_amount": 0.0,
@@ -143,9 +138,9 @@ class TrendingData:
old_state["total_amount"],
time_boost)
trending_score = old_state["trending_score"] + spike
self.claims[claim_hash] = {"total_amount": total_amount,
"trending_score": trending_score,
"changed": True}
self.claims[claim_id] = {"total_amount": total_amount,
"trending_score": trending_score,
"changed": True}
@@ -175,12 +170,13 @@ def test_trending():
# One global instance
# pylint: disable=C0103
trending_data = TrendingData()
f = open("trending_ar.log", "w")
f.close()
def run(db, height, final_height, recalculate_claim_hashes):
if height < final_height - 5*HALF_LIFE:
trending_log("Skipping AR trending at block {h}.\n".format(h=height))
return
start = time.time()
@@ -216,7 +212,7 @@ def run(db, height, final_height, recalculate_claim_hashes):
if not trending_data.initialised:
# On fresh launch
for row in db.execute("""
SELECT claim_hash, trending_mixed,
SELECT claim_id, trending_mixed,
(amount + support_amount)
AS total_amount
FROM claim;
@@ -225,7 +221,7 @@ def run(db, height, final_height, recalculate_claim_hashes):
trending_data.initialised = True
else:
for row in db.execute(f"""
SELECT claim_hash,
SELECT claim_id,
(amount + support_amount)
AS total_amount
FROM claim
@@ -252,7 +248,7 @@ def run(db, height, final_height, recalculate_claim_hashes):
trending_log("{n} scores to write...".format(n=len(the_list)))
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_hash=?;",
db.executemany("UPDATE claim SET trending_mixed=? WHERE claim_id=?;",
the_list)
trending_log("done.\n")
+4 -6
View File
@@ -94,7 +94,6 @@ class SQLDB:
create index if not exists claim_txo_hash_idx on claim (txo_hash);
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
create index if not exists claim_expiration_height_idx on claim (expiration_height);
create index if not exists claim_reposted_claim_hash_idx on claim (reposted_claim_hash);
"""
CREATE_SUPPORT_TABLE = """
@@ -242,10 +241,10 @@ class SQLDB:
streams, channels = {}, {}
if channel_hashes:
sql = query(
"SELECT repost.channel_hash, repost.reposted_claim_hash, target.claim_type "
"FROM claim as repost JOIN claim AS target ON (target.claim_hash=repost.reposted_claim_hash)", **{
'repost.reposted_claim_hash__is_not_null': 1,
'repost.channel_hash__in': channel_hashes
"SELECT claim.channel_hash, claim.reposted_claim_hash, reposted.claim_type "
"FROM claim JOIN claim AS reposted ON (reposted.claim_hash=claim.reposted_claim_hash)", **{
'claim.reposted_claim_hash__is_not_null': 1,
'claim.channel_hash__in': channel_hashes
}
)
for blocked_claim in self.execute(*sql):
@@ -361,7 +360,6 @@ class SQLDB:
if isinstance(fee.amount, Decimal):
claim_record['fee_amount'] = int(fee.amount*1000)
elif claim.is_repost:
claim_record['claim_type'] = CLAIM_TYPES['repost']
claim_record['reposted_claim_hash'] = claim.repost.reference.claim_hash
elif claim.is_channel:
claim_record['claim_type'] = CLAIM_TYPES['channel']
-20
View File
@@ -1,20 +0,0 @@
#!/usr/bin/env bash
# usage: update_dev_wallet_server.sh <host to update>
TARGET_HOST=$1
SCRIPTS_DIR=`dirname $0`
LBRY_DIR=`dirname $SCRIPTS_DIR`
# build the image
docker build -f $LBRY_DIR/docker/Dockerfile.wallet_server -t lbry/wallet-server:development $LBRY_DIR
IMAGE=`docker image inspect lbry/wallet-server:development | sed -n "s/^.*Id\":\s*\"sha256:\s*\(\S*\)\".*$/\1/p"`
# push the image to the server
ssh $TARGET_HOST docker image prune --force
docker save $IMAGE | ssh $TARGET_HOST docker load
ssh $TARGET_HOST docker tag $IMAGE lbry/wallet-server:development
# restart the wallet server
ssh $TARGET_HOST docker-compose down
ssh $TARGET_HOST WALLET_SERVER_TAG="development" docker-compose up -d
+2 -6
View File
@@ -124,11 +124,7 @@ def release(args):
current_version = Version.from_content(version_file)
print(f'Current Version: {current_version}')
if args.action == 'current':
new_version = current_version
else:
new_version = current_version.increment(args.action)
new_version = current_version.increment(args.action)
print(f' New Version: {new_version}')
previous_release = repo.release_from_tag(args.start_tag or current_version.tag)
@@ -245,7 +241,7 @@ def main():
parser.add_argument("--confirm", default=False, action="store_true",
help="without this flag, it will only print what it will do but will not actually do it")
parser.add_argument("--start-tag", help="custom starting tag for changelog generation")
parser.add_argument("action", choices=['test', 'current', 'major', 'minor', 'micro'])
parser.add_argument("action", choices=['test', 'major', 'minor', 'micro'])
args = parser.parse_args()
if args.action == "test":
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
import asyncio
import logging
import os
import sys
from lbry.wallet import database
def enable_logging():
root = logging.getLogger()
root.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s') # %(asctime)s - %(levelname)s -
handler.setFormatter(formatter)
root.addHandler(handler)
async def initialize_tables(queue, old_file, db):
await db.executescript("PRAGMA journal_mode=WAL")
await db.executescript("PRAGMA case_sensitive_like=true")
await db.executescript("CREATE TABLE IF NOT EXISTS claim_with_metadata "
"(claimID BLOB NOT NULL PRIMARY KEY, "
"name BLOB NOT NULL, "
"originalHeight INTEGER NOT NULL, "
"updateHeight INTEGER NOT NULL, "
"activationHeight INTEGER NOT NULL, "
"metadataField1 TEXT)")
await db.executescript(f"ATTACH '{old_file}' AS crd")
await db.create_function("NEEDS_UPDATE", 5, lambda id, name, original, update, activation:
queue.put_nowait({
"claimID": id, "name": name, "originalHeight": original,
"updateHeight": update, "activationHeight": activation
}))
await db.executescript("CREATE TEMP TRIGGER on_claim_insert AFTER INSERT ON crd.claim "
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(NEW.claimID, NEW.nodeName, "
"NEW.originalHeight, NEW.updateHeight, NEW.ActivationHeight); END")
await db.executescript("CREATE TEMP TRIGGER on_claim_update AFTER UPDATE ON crd.claim "
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(NEW.claimID, NEW.nodeName, "
"NEW.originalHeight, NEW.updateHeight, NEW.ActivationHeight); END")
await db.executescript("CREATE TEMP TRIGGER on_claim_delete AFTER DELETE ON crd.claim "
"FOR EACH ROW BEGIN SELECT NEEDS_UPDATE(OLD.claimID, OLD.nodeName, 0, 0, 0); END")
async def catch_up_on_missing(queue, db):
# items removed from crd are those that are in claim_with_metadata but not in crd
# items inserted into crd are those in crd but not in claim_with_metadata
# items updated are those that are in both but some columns don't match
await db.executescript("DELETE FROM claim_with_metadata WHERE claimID NOT IN"
"(SELECT c.claimID FROM crd.claim c)")
await db.executescript("INSERT INTO claim_with_metadata(claimID, name, originalHeight, updateHeight, activationHeight) "
"SELECT c.claimID, c.nodeName, c.originalHeight, c.updateHeight, c.activationHeight "
"FROM crd.claim c LEFT JOIN claim_with_metadata m ON c.claimID = m.claimID "
"WHERE m.claimID IS NULL OR c.nodeName != m.name OR c.updateHeight != m.updateHeight "
"OR c.activationHeight != m.activationHeight ON CONFLICT(claimID) DO UPDATE SET "
"name = excluded.name, originalHeight = excluded.originalHeight, "
"updateHeight = excluded.updateHeight, activationHeight = excluded.activationHeight")
async def remove_claim(claim, db):
await db.executescript("DELETE FROM claim_with_metadata WHERE claimID = :claimID", claim)
async def insert_update(claim, db):
# TODO: lookup and parse metadata here
await db.executescript("INSERT INTO claim_with_metadata(claimID, name, originalHeight, updateHeight, activationHeight) "
"VALUES(:claimID, :name, :originalHeight, :updateHeight, :activationHeight) "
"ON CONFLICT(claimID) DO UPDATE SET name = excluded.name, "
"originalHeight = excluded.originalHeight, updateHeight = excluded.updateHeight, "
"activationHeight = excluded.activationHeight", claim)
async def handle_changes():
old_file = os.path.expanduser("~/.lbrycrd/claims.sqlite")
new_file = os.path.expanduser("~/.lbrycrd/duplicate.sqlite")
db = database.AIOSQLite()
db = await db.connect(new_file) # super confusing that you have to use the return value here
queue = asyncio.Queue()
await initialize_tables(queue, old_file, db)
await catch_up_on_missing(queue, db)
while True:
claim = await queue.get()
if claim.updateHeight <= 0:
await remove_claim(claim, db)
else:
await insert_update(claim, db)
def main():
enable_logging()
try:
asyncio.run(handle_changes())
except KeyboardInterrupt:
logging.info("Process interrupted")
if __name__ == '__main__':
main()
+1 -1
View File
@@ -12,7 +12,7 @@ async def main():
try:
hostname = sys.argv[1]
except IndexError:
hostname = 'spv11.lbry.com'
hostname = 'spv1.lbry.com'
loop = asyncio.get_event_loop()
client = ClientSession(network=None, server=(hostname, 50001))
-1
View File
@@ -32,7 +32,6 @@ disable=
too-many-branches,
too-many-arguments,
too-many-statements,
too-many-nested-blocks,
too-many-public-methods,
too-many-instance-attributes,
protected-access,
+1 -1
View File
@@ -40,7 +40,7 @@ setup(
'colorama==0.3.7',
'distro==1.4.0',
'base58==1.0.0',
'cffi==1.13.2',
'cffi==1.12.1',
'cryptography==2.5',
'protobuf==3.6.1',
'msgpack==0.6.1',
@@ -309,14 +309,12 @@ class ClaimSearchCommand(ClaimTestCase):
octet = await self.stream_create()
video = await self.stream_create('chrome', file_path=self.video_file_name)
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
repost = await self.stream_repost(self.get_claim_id(image))
channel = await self.channel_create()
unknown = self.sout(tx)
# claim_type
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
await self.assertFindsClaims([channel], claim_type='channel')
await self.assertFindsClaims([repost], claim_type='repost')
# stream_type
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
@@ -324,7 +322,7 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([image], stream_types=['image'])
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
# media_type
# stream_type
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
await self.assertFindsClaims([video], media_types=['video/mp4'])
await self.assertFindsClaims([image], media_types=['image/png'])
@@ -386,25 +384,7 @@ class ClaimSearchCommand(ClaimTestCase):
class ClaimCommands(ClaimTestCase):
async def test_claim_list_type_filtering(self):
await self.channel_create()
await self.stream_create()
r = await self.claim_list(claim_type='channel')
self.assertEqual(1, len(r))
self.assertEqual('channel', r[0]['value_type'])
r = await self.claim_list(claim_type='stream')
self.assertEqual(1, len(r))
self.assertEqual('stream', r[0]['value_type'])
r = await self.claim_list(claim_type=['stream', 'channel'])
self.assertEqual(2, len(r))
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
async def test_claim_stream_channel_list_with_resolve(self):
self.assertListEqual([], await self.claim_list(resolve=True))
await self.channel_create()
await self.stream_create()
@@ -425,10 +405,10 @@ class ClaimCommands(ClaimTestCase):
await self.ledger.wait(channel_tx)
r = await self.claim_list(resolve=True)
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
self.assertEqual('not_found', r[0]['meta']['error']['name'])
self.assertTrue(r[1]['meta']['is_controlling'])
r = await self.channel_list(resolve=True)
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
self.assertEqual('not_found', r[0]['meta']['error']['name'])
self.assertTrue(r[1]['meta']['is_controlling'])
# confirm it
@@ -450,10 +430,10 @@ class ClaimCommands(ClaimTestCase):
await self.ledger.wait(stream_tx)
r = await self.claim_list(resolve=True)
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
self.assertEqual('not_found', r[0]['meta']['error']['name'])
self.assertTrue(r[1]['meta']['is_controlling'])
r = await self.stream_list(resolve=True)
self.assertEqual('NOT_FOUND', r[0]['meta']['error']['name'])
self.assertEqual('not_found', r[0]['meta']['error']['name'])
self.assertTrue(r[1]['meta']['is_controlling'])
# confirm it
@@ -865,26 +845,18 @@ class StreamCommands(ClaimTestCase):
# search for blocked content directly
result = await self.out(self.daemon.jsonrpc_claim_search(name='bad_content'))
blocked = result['blocked']
self.assertEqual([], result['items'])
self.assertEqual(1, blocked['total'])
self.assertEqual(1, len(blocked['channels']))
self.assertEqual(1, blocked['channels'][0]['blocked'])
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
self.assertEqual({"channels": {filtering_channel_id: 1}, "total": 1}, result['blocked'])
# search channel containing blocked content
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel'))
blocked = result['blocked']
self.assertEqual(1, len(result['items']))
self.assertEqual(1, blocked['total'])
self.assertEqual(1, len(blocked['channels']))
self.assertEqual(1, blocked['channels'][0]['blocked'])
self.assertTrue(blocked['channels'][0]['channel']['short_url'].startswith('lbry://@filtering#'))
self.assertEqual({"channels": {filtering_channel_id: 1}, "total": 1}, result['blocked'])
# content was filtered by not_tag before censoring
result = await self.out(self.daemon.jsonrpc_claim_search(channel='@some_channel', not_tags=["good", "bad"]))
self.assertEqual(0, len(result['items']))
self.assertEqual({"channels": [], "total": 0}, result['blocked'])
self.assertEqual({"channels": {}, "total": 0}, result['blocked'])
blocking_channel_id = self.get_claim_id(
await self.channel_create('@blocking', '1.0')
@@ -902,9 +874,8 @@ class StreamCommands(ClaimTestCase):
# blocked content is not resolveable
result = await self.out(self.daemon.jsonrpc_resolve('lbry://@some_channel/bad_content'))
error = result['lbry://@some_channel/bad_content']['error']
self.assertEqual(error['name'], 'BLOCKED')
self.assertTrue(error['name'], 'blocked')
self.assertTrue(error['text'].startswith("Resolve of 'lbry://@some_channel/bad_content' was censored"))
self.assertTrue(error['censor']['short_url'].startswith('lbry://@blocking#'))
async def test_publish_updates_file_list(self):
tx = await self.stream_create(title='created')
@@ -15,7 +15,7 @@ class BaseResolveTestCase(CommandTestCase):
other = (await self.resolve(name))[name]
if claim_id is None:
self.assertIn('error', other)
self.assertEqual(other['error']['name'], 'NOT_FOUND')
self.assertEqual(other['error']['name'], 'not_found')
else:
self.assertEqual(claim_id, other['claim_id'])
@@ -186,7 +186,7 @@ class ResolveCommand(BaseResolveTestCase):
self.assertEqual(response, {
'lbry://@abc/on-channel-claim': {
'error': {
'name': 'NOT_FOUND',
'name': 'not_found',
'text': 'Could not find claim at "lbry://@abc/on-channel-claim".',
}
}
@@ -265,7 +265,7 @@ class ResolveCommand(BaseResolveTestCase):
self.assertEqual(response, {
'@olds/bad_example': {
'error': {
'name': 'NOT_FOUND',
'name': 'not_found',
'text': 'Could not find claim at "@olds/bad_example".',
}
}
@@ -2,7 +2,7 @@ import logging
import pathlib
import time
from ..blockchain.test_claim_commands import ClaimTestCase
from .test_claim_commands import ClaimTestCase
from lbry.conf import TranscodeConfig
from lbry.file_analysis import VideoFileAnalyzer
+2 -2
View File
@@ -88,7 +88,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
self.assertEqual(
response['lbry://@spam/hovercraft'],
{'error': {
'name': 'NOT_FOUND',
'name': 'not_found',
'text': 'Could not find claim at "lbry://@spam/hovercraft".'
}}
)
@@ -192,7 +192,7 @@ class EpicAdventuresOfChris45(CommandTestCase):
self.assertEqual(
response[uri],
{'error': {
'name': 'NOT_FOUND',
'name': 'not_found',
'text': f'Could not find claim at "{uri}".'
}}
)
@@ -365,7 +365,7 @@ class CommentCommands(CommandTestCase):
async def test04_comment_abandons(self):
rswanson = (await self.channel_create('@RonSwanson'))['outputs'][0]
stream = (await self.stream_create('Pawnee_Tow_Hall_of_Fame_by_Leslie_Knope'))['outputs'][0]
stream = (await self.stream_create('Pawnee Town Hall of Fame by Leslie Knope'))['outputs'][0]
comment = await self.daemon.jsonrpc_comment_create(
comment='KNOPE! WHAT DID I TELL YOU ABOUT PUTTING MY INFORMATION UP LIKE THAT',
claim_id=stream['claim_id'],
@@ -383,7 +383,7 @@ class CommentCommands(CommandTestCase):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
stream = await self.stream_create('Cool Lamps to Sit On', channel_id=moth_id)
claim_id = stream['outputs'][0]['claim_id']
comment1 = await self.daemon.jsonrpc_comment_create(
@@ -440,7 +440,7 @@ class CommentCommands(CommandTestCase):
moth = (await self.channel_create('@InconspicuousMoth'))['outputs'][0]
bee = (await self.channel_create('@LazyBumblebee'))['outputs'][0]
moth_id = moth['claim_id']
stream = await self.stream_create('Cool_Lamps_to_Sit_On', channel_id=moth_id)
stream = await self.stream_create('Cool Lamps to Sit On', channel_id=moth_id)
claim_id = stream['outputs'][0]['claim_id']
hidden_comment = await self.daemon.jsonrpc_comment_create(
comment='Who on earth would want to sit around on a lamp all day',
@@ -21,5 +21,5 @@ class SettingsManagement(CommandTestCase):
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('server', 50001))
setting = self.daemon.jsonrpc_settings_clear('lbryum_servers')
self.assertEqual(setting['lbryum_servers'][0], ('spv11.lbry.com', 50001))
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv11.lbry.com', 50001))
self.assertEqual(setting['lbryum_servers'][0], ('spv1.lbry.com', 50001))
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv1.lbry.com', 50001))
+4 -4
View File
@@ -57,6 +57,10 @@ class TestURLParsing(unittest.TestCase):
url(f'lbry://@test#{claim_id}/stuff', channel_name='@test', channel_claim_id=claim_id, stream_name='stuff')
# unicode regex edges
_url = lambda name: url(name, stream_name=name)
_url('\u0009')
_url('\u000a')
_url('\u000d')
_url('\u0020')
_url('\uD799')
_url('\uE000')
_url('\uFFFD')
@@ -75,10 +79,6 @@ class TestURLParsing(unittest.TestCase):
fail("lbry://\uDFFE")
fail("lbry://\uFFFF")
fail("lbry://;")
fail("lbry://no\ttab")
fail("lbry://no space")
fail("lbry://no\rcr")
fail("lbry://no\new\nline")
fail("lbry://\"")
fail("lbry://\\")
fail("lbry:///")
-12
View File
@@ -98,12 +98,6 @@ class TestQueryBuilder(unittest.TestCase):
'txo_name__in0_1': 'def456'
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.name__in': {'abc123'}}),
('txo.name = :txo_name__in0', {
'txo_name__in0': 'abc123',
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.age__in': 'SELECT age from ages_table'}),
('txo.age IN (SELECT age from ages_table)', {})
@@ -124,12 +118,6 @@ class TestQueryBuilder(unittest.TestCase):
'txo_name__not_in0_1': 'def456'
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.name__not_in': ('abc123',)}),
('txo.name != :txo_name__not_in0', {
'txo_name__not_in0': 'abc123',
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.age__not_in': 'SELECT age from ages_table'}),
('txo.age NOT IN (SELECT age from ages_table)', {})