Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0e9ef69bd3 | ||
|
|
5c6d63c93d | ||
|
|
c8c10d2bb0 | ||
|
|
f4d3e9ea6a | ||
|
|
3713d3488d | ||
|
|
9b46d03c91 | ||
|
|
fe4e738580 | ||
|
|
205b0c4263 | ||
|
|
78eef25f1a | ||
|
|
c9305fd070 | ||
|
|
20774280b9 | ||
|
|
74bd2557af | ||
|
|
605fc8ecd8 | ||
|
|
6b745c53dc | ||
|
|
98c60b38fc | ||
|
|
be4515ec9a | ||
|
|
dcb1b64696 |
No files matched your search
+3
-2
@@ -84,14 +84,15 @@ test:json-api:
|
||||
|
||||
build:linux:
|
||||
extends: .build
|
||||
image: ubuntu:18.04 # cant use python3.7 image - binary won't run on ubuntu
|
||||
image: ubuntu:16.04
|
||||
variables:
|
||||
OS: linux
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends software-properties-common zip curl build-essential
|
||||
- add-apt-repository -y ppa:deadsnakes/ppa
|
||||
- apt-get install -y --no-install-recommends python3.7-dev python3-setuptools python3-wheel
|
||||
- apt-get update
|
||||
- apt-get install -y --no-install-recommends python3.7-dev
|
||||
- python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7
|
||||
|
||||
build:mac:
|
||||
|
||||
@@ -297,6 +297,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
def __init__(self, conf: Config, component_manager: typing.Optional[ComponentManager] = None):
|
||||
self.conf = conf
|
||||
self.platform_info = system_info.get_platform()
|
||||
self._video_file_analyzer = VideoFileAnalyzer(conf)
|
||||
self._node_id = None
|
||||
self._installation_id = None
|
||||
@@ -307,7 +308,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
skip_components=conf.components_to_skip or []
|
||||
)
|
||||
self.component_startup_task = None
|
||||
self._connection_status: typing.Tuple[float, bool] = [self.component_manager.loop.time(), False]
|
||||
self._connection_status: typing.Tuple[float, bool] = (self.component_manager.loop.time(), False)
|
||||
|
||||
logging.getLogger('aiohttp.access').setLevel(logging.WARN)
|
||||
rpc_app = web.Application()
|
||||
@@ -449,7 +450,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
async def start(self):
|
||||
log.info("Starting LBRYNet Daemon")
|
||||
log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2))
|
||||
log.info("Platform: %s", json.dumps(system_info.get_platform(), indent=2))
|
||||
log.info("Platform: %s", json.dumps(self.platform_info, indent=2))
|
||||
await self.analytics_manager.send_server_startup()
|
||||
await self.rpc_runner.setup()
|
||||
await self.streaming_runner.setup()
|
||||
@@ -894,9 +895,7 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
'build': (str) "dev" | "qa" | "rc" | "release",
|
||||
}
|
||||
"""
|
||||
platform_info = system_info.get_platform()
|
||||
log.info("Get version info: %s", json.dumps(platform_info))
|
||||
return platform_info
|
||||
return self.platform_info
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_resolve(self, urls: typing.Union[str, list], wallet_id=None):
|
||||
@@ -2180,7 +2179,12 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
return paginate_rows(claims, claim_count, page, page_size, claim_type=claim_type, resolve=resolve)
|
||||
|
||||
@requires(WALLET_COMPONENT)
|
||||
async def jsonrpc_claim_search(self, **kwargs):
|
||||
async def jsonrpc_claim_search(self, claim_ids=None, channel_ids=None, not_channel_ids=None,
|
||||
has_channel_signature=False, valid_channel_signature=False,
|
||||
invalid_channel_signature=False, is_controlling=False, stream_types=None,
|
||||
media_types=None, any_tags=None, all_tags=None, not_tags=None, any_languages=None,
|
||||
all_languages=None, not_languages=None, any_locations=None, all_locations=None,
|
||||
not_locations=None, order_by=None, **kwargs):
|
||||
"""
|
||||
Search for stream and channel claims on the blockchain.
|
||||
|
||||
@@ -2317,23 +2321,50 @@ class Daemon(metaclass=JSONRPCServerType):
|
||||
|
||||
Returns: {Paginated[Output]}
|
||||
"""
|
||||
wallet = self.wallet_manager.get_wallet_or_default(kwargs.pop('wallet_id', None))
|
||||
if {'claim_id', 'claim_ids'}.issubset(kwargs):
|
||||
|
||||
_kwargs = {}
|
||||
_kwargs.update(kwargs)
|
||||
_kwargs.update(
|
||||
{
|
||||
'claim_ids': claim_ids or [],
|
||||
'channel_ids': channel_ids or [],
|
||||
'not_channel_ids': not_channel_ids or [],
|
||||
'has_channel_signature': has_channel_signature,
|
||||
'valid_channel_signature': valid_channel_signature,
|
||||
'invalid_channel_signature': invalid_channel_signature,
|
||||
'is_controlling': is_controlling,
|
||||
'stream_types': stream_types or [],
|
||||
'media_types': media_types or [],
|
||||
'any_tags': any_tags or [],
|
||||
'all_tags': all_tags or [],
|
||||
'not_tags': not_tags or [],
|
||||
'any_languages': any_languages or [],
|
||||
'all_languages': all_languages or [],
|
||||
'not_languages': not_languages or [],
|
||||
'any_locations': any_locations or [],
|
||||
'all_locations': all_locations or [],
|
||||
'not_locations': not_locations or [],
|
||||
'order_by': order_by or []
|
||||
}
|
||||
)
|
||||
wallet = self.wallet_manager.get_wallet_or_default(_kwargs.pop('wallet_id', None))
|
||||
if {'claim_id', 'claim_ids'}.issubset(_kwargs):
|
||||
raise ValueError("Only 'claim_id' or 'claim_ids' is allowed, not both.")
|
||||
if kwargs.pop('valid_channel_signature', False):
|
||||
kwargs['signature_valid'] = 1
|
||||
if kwargs.pop('invalid_channel_signature', False):
|
||||
kwargs['signature_valid'] = 0
|
||||
page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
|
||||
kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
|
||||
txos, blocked, _, total = await self.ledger.claim_search(wallet.accounts, **kwargs)
|
||||
if _kwargs.pop('valid_channel_signature', False):
|
||||
_kwargs['signature_valid'] = 1
|
||||
if _kwargs.pop('invalid_channel_signature', False):
|
||||
_kwargs['signature_valid'] = 0
|
||||
|
||||
page_num, page_size = abs(_kwargs.pop('page', 1)), min(abs(_kwargs.pop('page_size', DEFAULT_PAGE_SIZE)), 50)
|
||||
_kwargs.update({'offset': page_size * (page_num - 1), 'limit': page_size})
|
||||
txos, blocked, _, total = await self.ledger.claim_search(wallet.accounts, **_kwargs)
|
||||
result = {
|
||||
"items": txos,
|
||||
"blocked": blocked,
|
||||
"page": page_num,
|
||||
"page_size": page_size
|
||||
}
|
||||
if not kwargs.pop('no_totals', False):
|
||||
if not _kwargs.pop('no_totals', False):
|
||||
result['total_pages'] = int((total + (page_size - 1)) / page_size)
|
||||
result['total_items'] = total
|
||||
return result
|
||||
|
||||
+11
-7
@@ -134,16 +134,20 @@ def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
|
||||
col, op = col[:-len('__not_like')], 'NOT LIKE'
|
||||
elif key.endswith('__in') or key.endswith('__not_in'):
|
||||
if key.endswith('__in'):
|
||||
col, op = col[:-len('__in')], 'IN'
|
||||
col, op, one_val_op = col[:-len('__in')], 'IN', '='
|
||||
else:
|
||||
col, op = col[:-len('__not_in')], 'NOT IN'
|
||||
col, op, one_val_op = col[:-len('__not_in')], 'NOT IN', '!='
|
||||
if constraint:
|
||||
if isinstance(constraint, (list, set, tuple)):
|
||||
keys = []
|
||||
for i, val in enumerate(constraint):
|
||||
keys.append(f':{key}{tag}_{i}')
|
||||
values[f'{key}{tag}_{i}'] = val
|
||||
sql.append(f'{col} {op} ({", ".join(keys)})')
|
||||
if len(constraint) == 1:
|
||||
values[f'{key}{tag}'] = next(iter(constraint))
|
||||
sql.append(f'{col} {one_val_op} :{key}{tag}')
|
||||
else:
|
||||
keys = []
|
||||
for i, val in enumerate(constraint):
|
||||
keys.append(f':{key}{tag}_{i}')
|
||||
values[f'{key}{tag}_{i}'] = val
|
||||
sql.append(f'{col} {op} ({", ".join(keys)})')
|
||||
elif isinstance(constraint, str):
|
||||
sql.append(f'{col} {op} ({constraint})')
|
||||
else:
|
||||
|
||||
@@ -630,7 +630,8 @@ class Ledger(metaclass=LedgerRegistry):
|
||||
raise asyncio.TimeoutError('Timed out waiting for transaction.')
|
||||
|
||||
async def _inflate_outputs(self, query, accounts) -> Tuple[List[Output], dict, int, int]:
|
||||
outputs = Outputs.from_base64(await query)
|
||||
encoded_outputs = await query
|
||||
outputs = Outputs.from_base64(encoded_outputs or b'') # TODO: why is the server returning None?
|
||||
txs = []
|
||||
if len(outputs.txs) > 0:
|
||||
txs: List[Transaction] = await asyncio.gather(*(
|
||||
|
||||
@@ -164,6 +164,7 @@ class BlockProcessor:
|
||||
# is consistent with self.height
|
||||
self.state_lock = asyncio.Lock()
|
||||
|
||||
self.search_cache = {}
|
||||
async def run_in_thread_with_lock(self, func, *args):
|
||||
# Run in a thread to prevent blocking. Shielded so that
|
||||
# cancellations from shutdown don't lose work - when the task
|
||||
@@ -191,6 +192,8 @@ class BlockProcessor:
|
||||
if hprevs == chain:
|
||||
start = time.perf_counter()
|
||||
await self.run_in_thread_with_lock(self.advance_blocks, blocks)
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
await self._maybe_flush()
|
||||
processed_time = time.perf_counter() - start
|
||||
BLOCK_COUNT.set(self.height)
|
||||
@@ -729,7 +732,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.logger.info(f"LbryumX Block Processor - Validating signatures: {self.should_validate_signatures}")
|
||||
self.sql: SQLDB = self.db.sql
|
||||
self.timer = Timer('BlockProcessor')
|
||||
self.search_cache = {}
|
||||
|
||||
def advance_blocks(self, blocks):
|
||||
self.sql.begin()
|
||||
@@ -744,8 +746,6 @@ class LBRYBlockProcessor(BlockProcessor):
|
||||
self.timer.run(self.sql.execute, self.sql.SEARCH_INDEXES, timer_name='executing SEARCH_INDEXES')
|
||||
if self.env.individual_tag_indexes:
|
||||
self.timer.run(self.sql.execute, self.sql.TAG_INDEXES, timer_name='executing TAG_INDEXES')
|
||||
for cache in self.search_cache.values():
|
||||
cache.clear()
|
||||
|
||||
def advance_txs(self, height, txs, header):
|
||||
timer = self.timer.sub_timers['advance_blocks']
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
CLAIM_TYPES = {
|
||||
'stream': 1,
|
||||
'channel': 2,
|
||||
'repost': 3
|
||||
}
|
||||
|
||||
STREAM_TYPES = {
|
||||
|
||||
@@ -266,7 +266,7 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
else:
|
||||
constraints['claim.claim_id__like'] = f'{claim_id[:40]}%'
|
||||
elif 'claim_ids' in constraints:
|
||||
constraints['claim.claim_id__in'] = constraints.pop('claim_ids')
|
||||
constraints['claim.claim_id__in'] = set(constraints.pop('claim_ids'))
|
||||
|
||||
if 'reposted_claim_id' in constraints:
|
||||
constraints['claim.reposted_claim_hash'] = unhexlify(constraints.pop('reposted_claim_id'))[::-1]
|
||||
@@ -282,15 +282,15 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
if 'channel_ids' in constraints:
|
||||
channel_ids = constraints.pop('channel_ids')
|
||||
if channel_ids:
|
||||
constraints['claim.channel_hash__in'] = [
|
||||
constraints['claim.channel_hash__in'] = {
|
||||
unhexlify(cid)[::-1] for cid in channel_ids
|
||||
]
|
||||
}
|
||||
if 'not_channel_ids' in constraints:
|
||||
not_channel_ids = constraints.pop('not_channel_ids')
|
||||
if not_channel_ids:
|
||||
not_channel_ids_binary = [
|
||||
not_channel_ids_binary = {
|
||||
unhexlify(ncid)[::-1] for ncid in not_channel_ids
|
||||
]
|
||||
}
|
||||
if constraints.get('has_channel_signature', False):
|
||||
constraints['claim.channel_hash__not_in'] = not_channel_ids_binary
|
||||
else:
|
||||
@@ -320,13 +320,13 @@ def claims_query(cols, for_count=False, **constraints) -> Tuple[str, Dict]:
|
||||
if 'stream_types' in constraints:
|
||||
stream_types = constraints.pop('stream_types')
|
||||
if stream_types:
|
||||
constraints['claim.stream_type__in'] = [
|
||||
constraints['claim.stream_type__in'] = {
|
||||
STREAM_TYPES[stream_type] for stream_type in stream_types
|
||||
]
|
||||
}
|
||||
if 'media_types' in constraints:
|
||||
media_types = constraints.pop('media_types')
|
||||
if media_types:
|
||||
constraints['claim.media_type__in'] = media_types
|
||||
constraints['claim.media_type__in'] = set(media_types)
|
||||
|
||||
if 'fee_currency' in constraints:
|
||||
constraints['claim.fee_currency'] = constraints.pop('fee_currency').lower()
|
||||
@@ -418,8 +418,6 @@ def search(constraints) -> Tuple[List, List, int, int, Censor]:
|
||||
total = count_claims(**constraints)
|
||||
constraints['offset'] = abs(constraints.get('offset', 0))
|
||||
constraints['limit'] = min(abs(constraints.get('limit', 10)), 50)
|
||||
if 'order_by' not in constraints:
|
||||
constraints['order_by'] = ["claim_hash"]
|
||||
context = ctx.get()
|
||||
search_censor = context.get_search_censor()
|
||||
txo_rows = search_claims(search_censor, **constraints)
|
||||
|
||||
@@ -94,6 +94,7 @@ class SQLDB:
|
||||
create index if not exists claim_txo_hash_idx on claim (txo_hash);
|
||||
create index if not exists claim_activation_height_idx on claim (activation_height, claim_hash);
|
||||
create index if not exists claim_expiration_height_idx on claim (expiration_height);
|
||||
create index if not exists claim_reposted_claim_hash_idx on claim (reposted_claim_hash);
|
||||
"""
|
||||
|
||||
CREATE_SUPPORT_TABLE = """
|
||||
@@ -360,6 +361,7 @@ class SQLDB:
|
||||
if isinstance(fee.amount, Decimal):
|
||||
claim_record['fee_amount'] = int(fee.amount*1000)
|
||||
elif claim.is_repost:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['repost']
|
||||
claim_record['reposted_claim_hash'] = claim.repost.reference.claim_hash
|
||||
elif claim.is_channel:
|
||||
claim_record['claim_type'] = CLAIM_TYPES['channel']
|
||||
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# usage: update_dev_wallet_server.sh <host to update>
|
||||
TARGET_HOST=$1
|
||||
|
||||
SCRIPTS_DIR=`dirname $0`
|
||||
LBRY_DIR=`dirname $SCRIPTS_DIR`
|
||||
|
||||
# build the image
|
||||
docker build -f $LBRY_DIR/docker/Dockerfile.wallet_server -t lbry/wallet-server:development $LBRY_DIR
|
||||
IMAGE=`docker image inspect lbry/wallet-server:development | sed -n "s/^.*Id\":\s*\"sha256:\s*\(\S*\)\".*$/\1/p"`
|
||||
|
||||
# push the image to the server
|
||||
ssh $TARGET_HOST docker image prune --force
|
||||
docker save $IMAGE | ssh $TARGET_HOST docker load
|
||||
ssh $TARGET_HOST docker tag $IMAGE lbry/wallet-server:development
|
||||
|
||||
# restart the wallet server
|
||||
ssh $TARGET_HOST docker-compose down
|
||||
ssh $TARGET_HOST WALLET_SERVER_TAG="development" docker-compose up -d
|
||||
@@ -32,6 +32,7 @@ disable=
|
||||
too-many-branches,
|
||||
too-many-arguments,
|
||||
too-many-statements,
|
||||
too-many-nested-blocks,
|
||||
too-many-public-methods,
|
||||
too-many-instance-attributes,
|
||||
protected-access,
|
||||
|
||||
@@ -309,12 +309,14 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
octet = await self.stream_create()
|
||||
video = await self.stream_create('chrome', file_path=self.video_file_name)
|
||||
image = await self.stream_create('blank-image', data=self.image_data, suffix='.png')
|
||||
repost = await self.stream_repost(self.get_claim_id(image))
|
||||
channel = await self.channel_create()
|
||||
unknown = self.sout(tx)
|
||||
|
||||
# claim_type
|
||||
await self.assertFindsClaims([image, video, octet, unknown], claim_type='stream')
|
||||
await self.assertFindsClaims([channel], claim_type='channel')
|
||||
await self.assertFindsClaims([repost], claim_type='repost')
|
||||
|
||||
# stream_type
|
||||
await self.assertFindsClaims([octet, unknown], stream_types=['binary'])
|
||||
@@ -322,7 +324,7 @@ class ClaimSearchCommand(ClaimTestCase):
|
||||
await self.assertFindsClaims([image], stream_types=['image'])
|
||||
await self.assertFindsClaims([image, video], stream_types=['video', 'image'])
|
||||
|
||||
# stream_type
|
||||
# media_type
|
||||
await self.assertFindsClaims([octet, unknown], media_types=['application/octet-stream'])
|
||||
await self.assertFindsClaims([video], media_types=['video/mp4'])
|
||||
await self.assertFindsClaims([image], media_types=['image/png'])
|
||||
@@ -401,6 +403,8 @@ class ClaimCommands(ClaimTestCase):
|
||||
self.assertEqual({'stream', 'channel'}, {c['value_type'] for c in r})
|
||||
|
||||
async def test_claim_stream_channel_list_with_resolve(self):
|
||||
self.assertListEqual([], await self.claim_list(resolve=True))
|
||||
|
||||
await self.channel_create()
|
||||
await self.stream_create()
|
||||
|
||||
|
||||
@@ -98,6 +98,12 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
'txo_name__in0_1': 'def456'
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.name__in': {'abc123'}}),
|
||||
('txo.name = :txo_name__in0', {
|
||||
'txo_name__in0': 'abc123',
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.age__in': 'SELECT age from ages_table'}),
|
||||
('txo.age IN (SELECT age from ages_table)', {})
|
||||
@@ -118,6 +124,12 @@ class TestQueryBuilder(unittest.TestCase):
|
||||
'txo_name__not_in0_1': 'def456'
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.name__not_in': ('abc123',)}),
|
||||
('txo.name != :txo_name__not_in0', {
|
||||
'txo_name__not_in0': 'abc123',
|
||||
})
|
||||
)
|
||||
self.assertTupleEqual(
|
||||
constraints_to_sql({'txo.age__not_in': 'SELECT age from ages_table'}),
|
||||
('txo.age NOT IN (SELECT age from ages_table)', {})
|
||||
|
||||
Reference in new issue
Block a user