Compare commits

...
Author SHA1 Message Date
Jack Robison 0e9ef69bd3 fix claim_search rpc and cli behaving differently
-don't autopopulate order_by in the server, instead populate an empty list in the client (which is what cli already did)
2020-02-13 17:10:26 -05:00
Jack Robison 5c6d63c93d Merge pull request #2803 from lbryio/fix-2802
fix claim apis failing with --resolve if no claims are present
2020-02-13 15:15:56 -05:00
Jack Robison c8c10d2bb0 fix claim apis failing with --resolve if no claims are present
fixes https://github.com/lbryio/lbry-sdk/issues/2802
2020-02-13 14:58:51 -05:00
Jack Robison f4d3e9ea6a add development wallet server deploy script 2020-02-13 13:54:46 -05:00
Jack Robison 3713d3488d Merge pull request #2800 from lbryio/fix-2746
Fix race condition setting/clearing the claim_search and resolve cache
2020-02-13 11:39:27 -05:00
Jack Robison 9b46d03c91 fix race between setting items in and clearing the resolve/claim_search cache 2020-02-13 11:17:21 -05:00
Jack Robison fe4e738580 Merge pull request #2798 from lbryio/repost-index
Add sqlite index for reposted claims
2020-02-13 10:33:36 -05:00
Jack Robison 205b0c4263 call get_platform only once 2020-02-13 10:04:50 -05:00
Jack Robison 78eef25f1a add reposted_claim_hash index
fixes https://github.com/lbryio/lbry-sdk/issues/2782
2020-02-12 17:39:57 -05:00
Lex Berezhny c9305fd070 Merge pull request #2796 from lbryio/xenial_build
linux build of LBRY SDK for Ubuntu 16.04 Xenial
2020-02-12 14:28:41 -05:00
Lex Berezhny 20774280b9 Merge pull request #2790 from lbryio/sql_in_for_single_value
SQL generation fix to handle IN operation for one value lists
2020-02-12 14:27:36 -05:00
Lex Berezhny 74bd2557af linux build of LBRY SDK for Ubuntu 16.05 Xenial 2020-02-12 14:20:06 -05:00
Lex Berezhny 605fc8ecd8 claim_search by --claim_type=repost support 2020-02-12 11:51:35 -05:00
Lex Berezhny 6b745c53dc make sure SQL IN works with python set 2020-02-12 11:41:32 -05:00
Lex Berezhny 98c60b38fc disable too-many-nested-blocks lint rule 2020-02-12 10:47:40 -05:00
Lex Berezhny be4515ec9a use sets for __in constraint values 2020-02-12 10:39:25 -05:00
Lex Berezhny dcb1b64696 SQL generation fix to handle IN operation for one value lists 2020-02-12 10:31:27 -05:00
Jack Robison 1190a60c7a v0.59.1 2020-02-11 16:56:37 -05:00
Jack Robison 82ecf207c6 Merge pull request #2789 from lbryio/update-default-servers
update default wallet servers
2020-02-11 16:36:31 -05:00
Jack Robison 84c7f1bf58 update test 2020-02-11 16:13:55 -05:00
Jack Robison 5ac453fe07 update default wallet servers 2020-02-11 13:51:15 -05:00
16 changed files with 128 additions and 53 deletions

No files matched your search

+3 -2
View File
@@ -84,14 +84,15 @@ test:json-api:
build:linux:
extends: .build
image: ubuntu:18.04 # cant use python3.7 image - binary won't run on ubuntu
image: ubuntu:16.04
variables:
OS: linux
before_script:
- apt-get update
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
- add-apt-repository -y ppa:deadsnakes/ppa
- apt-get install -y --no-install-recommends python3.7-dev python3-setuptools python3-wheel
- apt-get update
- apt-get install -y --no-install-recommends python3.7-dev
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
build:mac:
+1 -1
View File
@@ -1,2 +1,2 @@
__version__ = "0.59.0"
__version__ = "0.59.1"
version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name
+9 -9
View File
@@ -573,15 +573,15 @@ class Config(CLIConfig):
('reflector.lbry.com', 5566)
])
lbryum_servers = Servers("SPV wallet servers", [
('spv1.lbry.com', 50001),
('spv2.lbry.com', 50001),
('spv3.lbry.com', 50001),
('spv4.lbry.com', 50001),
('spv5.lbry.com', 50001),
('spv6.lbry.com', 50001),
('spv7.lbry.com', 50001),
('spv8.lbry.com', 50001),
('spv9.lbry.com', 50001),
('spv11.lbry.com', 50001),
('spv12.lbry.com', 50001),
('spv13.lbry.com', 50001),
('spv14.lbry.com', 50001),
('spv15.lbry.com', 50001),
('spv16.lbry.com', 50001),
('spv17.lbry.com', 50001),
('spv18.lbry.com', 50001),
('spv19.lbry.com', 50001),
])
known_dht_nodes = Servers("Known nodes for bootstrapping connection to the DHT", [
('lbrynet1.lbry.com', 4444), # US EAST
+47 -16
View File
@@ -297,6 +297,7 @@ class Daemon(metaclass=JSONRPCServerType):
def __init__(self, conf: Config, component_manager: typing.Optional[ComponentManager] = None):
self.conf = conf
self.platform_info = system_info.get_platform()
self._video_file_analyzer = VideoFileAnalyzer(conf)
self._node_id = None
self._installation_id = None
@@ -307,7 +308,7 @@ class Daemon(metaclass=JSONRPCServerType):
skip_components=conf.components_to_skip or []
)
self.component_startup_task = None
self._connection_status: typing.Tuple[float, bool] = [self.component_manager.loop.time(), False]
self._connection_status: typing.Tuple[float, bool] = (self.component_manager.loop.time(), False)
logging.getLogger('aiohttp.access').setLevel(logging.WARN)
rpc_app = web.Application()
@@ -449,7 +450,7 @@ class Daemon(metaclass=JSONRPCServerType):
async def start(self):
log.info("Starting LBRYNet Daemon")
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
log.info("Platform: %s", json.dumps(system_info.get_platform(), indent=2))
log.info("Platform: %s", json.dumps(self.platform_info, indent=2))
await self.analytics_manager.send_server_startup()
await self.rpc_runner.setup()
await self.streaming_runner.setup()
@@ -894,9 +895,7 @@ class Daemon(metaclass=JSONRPCServerType):
'build': (str) "dev" | "qa" | "rc" | "release",
}
"""
platform_info = system_info.get_platform()
log.info("Get version info: %s", json.dumps(platform_info))
return platform_info
return self.platform_info
@requires(WALLET_COMPONENT)
async def jsonrpc_resolve(self, urls: typing.Union[str, list], wallet_id=None):
@@ -2180,7 +2179,12 @@ class Daemon(metaclass=JSONRPCServerType):
return paginate_rows(claims, claim_count, page, page_size, claim_type=claim_type, resolve=resolve)
@requires(WALLET_COMPONENT)
async def jsonrpc_claim_search(self, **kwargs):
async def jsonrpc_claim_search(self, claim_ids=None, channel_ids=None, not_channel_ids=None,
has_channel_signature=False, valid_channel_signature=False,
invalid_channel_signature=False, is_controlling=False, stream_types=None,
media_types=None, any_tags=None, all_tags=None, not_tags=None, any_languages=None,
all_languages=None, not_languages=None, any_locations=None, all_locations=None,
not_locations=None, order_by=None, **kwargs):
"""
Search for stream and channel claims on the blockchain.
@@ -2317,23 +2321,50 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
wallet = self.wallet_manager.get_wallet_or_default(kwargs.pop('wallet_id', None))
if {'claim_id', 'claim_ids'}.issubset(kwargs):
_kwargs = {}
_kwargs.update(kwargs)
_kwargs.update(
{
'claim_ids': claim_ids or [],
'channel_ids': channel_ids or [],
'not_channel_ids': not_channel_ids or [],
'has_channel_signature': has_channel_signature,
'valid_channel_signature': valid_channel_signature,
'invalid_channel_signature': invalid_channel_signature,
'is_controlling': is_controlling,
'stream_types': stream_types or [],
'media_types': media_types or [],
'any_tags': any_tags or [],
'all_tags': all_tags or [],
'not_tags': not_tags or [],
'any_languages': any_languages or [],
'all_languages': all_languages or [],
'not_languages': not_languages or [],
'any_locations': any_locations or [],
'all_locations': all_locations or [],
'not_locations': not_locations or [],
'order_by': order_by or []
}
)
wallet = self.wallet_manager.get_wallet_or_default(_kwargs.pop('wallet_id', None))
if {'claim_id', 'claim_ids'}.issubset(_kwargs):
raise ValueError("Only 'claim_id' or 'claim_ids' is allowed, not both.")
if kwargs.pop('valid_channel_signature', False):
kwargs['signature_valid'] = 1
if kwargs.pop('invalid_channel_signature', False):
kwargs['signature_valid'] = 0
page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
txos, blocked, _, total = await self.ledger.claim_search(wallet.accounts, **kwargs)
if _kwargs.pop('valid_channel_signature', False):
_kwargs['signature_valid'] = 1
if _kwargs.pop('invalid_channel_signature', False):
_kwargs['signature_valid'] = 0
page_num, page_size = abs(_kwargs.pop('page', 1)), min(abs(_kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
_kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
txos, blocked, _, total = await self.ledger.claim_search(wallet.accounts, **_kwargs)
result = {
"items": txos,
"blocked": blocked,
"page": page_num,
"page_size": page_size
}
if not kwargs.pop('no_totals', False):
if not _kwargs.pop('no_totals', False):
result['total_pages'] = int((total + (page_size - 1)) / page_size)
result['total_items'] = total
return result
+11 -7
View File
@@ -134,16 +134,20 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
col, op = col[:-len('__not_like')], 'NOT LIKE'
elif key.endswith('__in') or key.endswith('__not_in'):
if key.endswith('__in'):
col, op = col[:-len('__in')], 'IN'
col, op, one_val_op = col[:-len('__in')], 'IN', '='
else:
col, op = col[:-len('__not_in')], 'NOT IN'
col, op, one_val_op = col[:-len('__not_in')], 'NOT IN', '!='
if constraint:
if isinstance(constraint, (list, set, tuple)):
keys = []
for i, val in enumerate(constraint):
keys.append(f':{key}{tag}_{i}')
values[f'{key}{tag}_{i}'] = val
sql.append(f'{col} {op} ({", ".join(keys)})')
if len(constraint) == 1:
values[f'{key}{tag}'] = next(iter(constraint))
sql.append(f'{col} {one_val_op} :{key}{tag}')
else:
keys = []
for i, val in enumerate(constraint):
keys.append(f':{key}{tag}_{i}')
values[f'{key}{tag}_{i}'] = val
sql.append(f'{col} {op} ({", ".join(keys)})')
elif isinstance(constraint, str):
sql.append(f'{col} {op} ({constraint})')
else:
+2 -1
View File
@@ -630,7 +630,8 @@ class Ledger(metaclass=LedgerRegistry):
raise asyncio.TimeoutError('Timed out waiting for transaction.')
async def _inflate_outputs(self, query, accounts) -> Tuple[List[Output], dict, int, int]:
outputs = Outputs.from_base64(await query)
encoded_outputs = await query
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
txs = []
if len(outputs.txs) > 0:
txs: List[Transaction] = await asyncio.gather(*(
+3 -3
View File
@@ -164,6 +164,7 @@ class BlockProcessor:
# is consistent with self.height
self.state_lock = asyncio.Lock()
self.search_cache = {}
async def run_in_thread_with_lock(self, func, *args):
# Run in a thread to prevent blocking. Shielded so that
# cancellations from shutdown don't lose work - when the task
@@ -191,6 +192,8 @@ class BlockProcessor:
if hprevs == chain:
start = time.perf_counter()
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
for cache in self.search_cache.values():
cache.clear()
await self._maybe_flush()
processed_time = time.perf_counter() - start
BLOCK_COUNT.set(self.height)
@@ -729,7 +732,6 @@ class LBRYBlockProcessor(BlockProcessor):
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
self.sql: SQLDB = self.db.sql
self.timer = Timer('BlockProcessor')
self.search_cache = {}
def advance_blocks(self, blocks):
self.sql.begin()
@@ -744,8 +746,6 @@ class LBRYBlockProcessor(BlockProcessor):
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
if self.env.individual_tag_indexes:
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
for cache in self.search_cache.values():
cache.clear()
def advance_txs(self, height, txs, header):
timer = self.timer.sub_timers['advance_blocks']
+1
View File
@@ -1,6 +1,7 @@
CLAIM_TYPES = {
'stream': 1,
'channel': 2,
'repost': 3
}
STREAM_TYPES = {
+8 -10
View File
@@ -266,7 +266,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
else:
constraints['claim.claim_id__like'] = f'{claim_id[:40]}%'
elif 'claim_ids' in constraints:
constraints['claim.claim_id__in'] = constraints.pop('claim_ids')
constraints['claim.claim_id__in'] = set(constraints.pop('claim_ids'))
if 'reposted_claim_id' in constraints:
constraints['claim.reposted_claim_hash'] = unhexlify(constraints.pop('reposted_claim_id'))[::-1]
@@ -282,15 +282,15 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
if 'channel_ids' in constraints:
channel_ids = constraints.pop('channel_ids')
if channel_ids:
constraints['claim.channel_hash__in'] = [
constraints['claim.channel_hash__in'] = {
unhexlify(cid)[::-1] for cid in channel_ids
]
}
if 'not_channel_ids' in constraints:
not_channel_ids = constraints.pop('not_channel_ids')
if not_channel_ids:
not_channel_ids_binary = [
not_channel_ids_binary = {
unhexlify(ncid)[::-1] for ncid in not_channel_ids
]
}
if constraints.get('has_channel_signature', False):
constraints['claim.channel_hash__not_in'] = not_channel_ids_binary
else:
@@ -320,13 +320,13 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
if 'stream_types' in constraints:
stream_types = constraints.pop('stream_types')
if stream_types:
constraints['claim.stream_type__in'] = [
constraints['claim.stream_type__in'] = {
STREAM_TYPES[stream_type] for stream_type in stream_types
]
}
if 'media_types' in constraints:
media_types = constraints.pop('media_types')
if media_types:
constraints['claim.media_type__in'] = media_types
constraints['claim.media_type__in'] = set(media_types)
if 'fee_currency' in constraints:
constraints['claim.fee_currency'] = constraints.pop('fee_currency').lower()
@@ -418,8 +418,6 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
total = count_claims(**constraints)
constraints['offset'] = abs(constraints.get('offset', 0))
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
if 'order_by' not in constraints:
constraints['order_by'] = ["claim_hash"]
context = ctx.get()
search_censor = context.get_search_censor()
txo_rows = search_claims(search_censor, **constraints)
+2
View File
@@ -94,6 +94,7 @@ class SQLDB:
create index if not exists claim_txo_hash_idx on claim (txo_hash);
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
create index if not exists claim_expiration_height_idx on claim (expiration_height);
create index if not exists claim_reposted_claim_hash_idx on claim (reposted_claim_hash);
"""
CREATE_SUPPORT_TABLE = """
@@ -360,6 +361,7 @@ class SQLDB:
if isinstance(fee.amount, Decimal):
claim_record['fee_amount'] = int(fee.amount*1000)
elif claim.is_repost:
claim_record['claim_type'] = CLAIM_TYPES['repost']
claim_record['reposted_claim_hash'] = claim.repost.reference.claim_hash
elif claim.is_channel:
claim_record['claim_type'] = CLAIM_TYPES['channel']
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
# usage: update_dev_wallet_server.sh <host to update>
TARGET_HOST=$1
SCRIPTS_DIR=`dirname $0`
LBRY_DIR=`dirname $SCRIPTS_DIR`
# build the image
docker build -f $LBRY_DIR/docker/Dockerfile.wallet_server -t lbry/wallet-server:development $LBRY_DIR
IMAGE=`docker image inspect lbry/wallet-server:development | sed -n "s/^.*Id\":\s*\"sha256:\s*\(\S*\)\".*$/\1/p"`
# push the image to the server
ssh $TARGET_HOST docker image prune --force
docker save $IMAGE | ssh $TARGET_HOST docker load
ssh $TARGET_HOST docker tag $IMAGE lbry/wallet-server:development
# restart the wallet server
ssh $TARGET_HOST docker-compose down
ssh $TARGET_HOST WALLET_SERVER_TAG="development" docker-compose up -d
+1 -1
View File
@@ -12,7 +12,7 @@ async def main():
try:
hostname = sys.argv[1]
except IndexError:
hostname = 'spv1.lbry.com'
hostname = 'spv11.lbry.com'
loop = asyncio.get_event_loop()
client = ClientSession(network=None, server=(hostname, 50001))
+1
View File
@@ -32,6 +32,7 @@ disable=
too-many-branches,
too-many-arguments,
too-many-statements,
too-many-nested-blocks,
too-many-public-methods,
too-many-instance-attributes,
protected-access,
@@ -309,12 +309,14 @@ class ClaimSearchCommand(ClaimTestCase):
octet = await self.stream_create()
video = await self.stream_create('chrome', file_path=self.video_file_name)
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
repost = await self.stream_repost(self.get_claim_id(image))
channel = await self.channel_create()
unknown = self.sout(tx)
# claim_type
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
await self.assertFindsClaims([channel], claim_type='channel')
await self.assertFindsClaims([repost], claim_type='repost')
# stream_type
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
@@ -322,7 +324,7 @@ class ClaimSearchCommand(ClaimTestCase):
await self.assertFindsClaims([image], stream_types=['image'])
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
# stream_type
# media_type
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
await self.assertFindsClaims([video], media_types=['video/mp4'])
await self.assertFindsClaims([image], media_types=['image/png'])
@@ -401,6 +403,8 @@ class ClaimCommands(ClaimTestCase):
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
async def test_claim_stream_channel_list_with_resolve(self):
self.assertListEqual([], await self.claim_list(resolve=True))
await self.channel_create()
await self.stream_create()
@@ -21,5 +21,5 @@ class SettingsManagement(CommandTestCase):
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('server', 50001))
setting = self.daemon.jsonrpc_settings_clear('lbryum_servers')
self.assertEqual(setting['lbryum_servers'][0], ('spv1.lbry.com', 50001))
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv1.lbry.com', 50001))
self.assertEqual(setting['lbryum_servers'][0], ('spv11.lbry.com', 50001))
self.assertEqual(self.daemon.jsonrpc_settings_get()['lbryum_servers'][0], ('spv11.lbry.com', 50001))
+12
View File
@@ -98,6 +98,12 @@ class TestQueryBuilder(unittest.TestCase):
'txo_name__in0_1': 'def456'
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.name__in': {'abc123'}}),
('txo.name = :txo_name__in0', {
'txo_name__in0': 'abc123',
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.age__in': 'SELECT age from ages_table'}),
('txo.age IN (SELECT age from ages_table)', {})
@@ -118,6 +124,12 @@ class TestQueryBuilder(unittest.TestCase):
'txo_name__not_in0_1': 'def456'
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.name__not_in': ('abc123',)}),
('txo.name != :txo_name__not_in0', {
'txo_name__not_in0': 'abc123',
})
)
self.assertTupleEqual(
constraints_to_sql({'txo.age__not_in': 'SELECT age from ages_table'}),
('txo.age NOT IN (SELECT age from ages_table)', {})