Compare commits

..
Author SHA1 Message Date
Victor Shyba 3a0affe8ba show status while downloading headers 2019-06-11 02:25:49 -03:00
Jack Robison 06a9770504 Merge pull request #2216 from lbryio/fix-connection-status-startup-error
fix connections status raising an error during startup
2019-06-07 13:55:54 -04:00
Jack Robison 0da43d3e05 fix connections status raising an error during startup 2019-06-07 12:14:05 -04:00
Lex Berezhny 077dbb3138 better handle claims which fail to decode 2019-06-07 08:42:17 -04:00
Jack Robison 3b8fd65fa9 Merge pull request #2212 from lbryio/add-torba-version-to-version-rpc
update `version` docstring, add `torba_version`
2019-06-05 20:59:58 -04:00
Jack Robison d0ddaa5518 update version docstring, add torba_version 2019-06-05 20:33:21 -04:00
Jack Robison a60ab97552 Merge pull request #2211 from lbryio/pin-aioupnp
pin aioupnp requirement to 0.0.12
2019-06-05 19:41:45 -04:00
Jack Robison 078d09976c pin aioupnp requirement 2019-06-05 18:50:36 -04:00
Jack Robison 3770558694 Merge pull request #2210 from lbryio/fix-wine-build
fix torba version on travis
2019-06-05 18:46:24 -04:00
Jack Robison 4409b8c4f8 fix torba version on travis 2019-06-05 18:19:56 -04:00
Lex Berezhny f5b7aaac18 v0.38.0rc6 2019-06-04 22:32:33 -04:00
Lex Berezhny 6dbb6b9079 fix claim expiration calculation 2019-06-04 22:28:35 -04:00
Lex Berezhny 16b774a69e add normalized to claims_in_channel index to make update claim faster 2019-06-04 22:28:35 -04:00
Lex Berezhny 72ea19532e added index specifically for claim_in_channel computation 2019-06-04 22:28:35 -04:00
Lex Berezhny ad24946af3 add improved profiling to validate_channel_signatures 2019-06-04 22:28:35 -04:00
Lex Berezhny b676907fd7 added index for claim expiration_height 2019-06-04 22:28:35 -04:00
Lex Berezhny 6965971613 delete claims and re-calculate winner when claim is expired 2019-06-04 22:28:35 -04:00
Lex Berezhny 6651111522 pylint 2019-06-04 18:44:21 -04:00
Lex Berezhny d8630a02d9 Strings setting works like Server from CLI 2019-06-04 18:44:21 -04:00
Lex Berezhny 69345b4b0e integrationt tests fixed 2019-06-04 16:04:46 -04:00
Lex Berezhny f23aea9951 better handling for claims in abandoned channels 2019-06-04 16:04:46 -04:00
Lex Berezhny f96c46e84a claim search fixes for filtering with valid_channel_signatures 2019-06-04 16:04:46 -04:00
Alex Grintsvayg bae2939943 change a few instances of lbry.io to lbry.com. fixes #2135 2019-06-04 10:23:04 -04:00
Alex Grintsvayg 7063138e3c small improvements from our cli investigation 2019-06-04 09:39:33 -04:00
28 changed files with 229 additions and 98 deletions

No files matched your search

+1 -1
View File
@@ -7,7 +7,7 @@ Before reporting any issues, please make sure that you're using the latest versi
- App: https://github.com/lbryio/lbry-desktop/releases
- Daemon: https://github.com/lbryio/lbry/releases
We are also available on Discord at https://chat.lbry.io
We are also available on Discord at https://chat.lbry.com
-->
+6 -4
View File
@@ -4,7 +4,9 @@ language: python
python: "3.7"
env:
- TORBA=master
global:
# must also be updated in wine_build.sh
- TORBA=master
jobs:
include:
@@ -59,7 +61,7 @@ jobs:
- docker pull lbry/pyinstaller34_32bits:py371
script:
- python scripts/set_build.py
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh ${TORBA}
- docker run -v "$(pwd):/src/lbry" lbry/pyinstaller34_32bits:py371 lbry/scripts/wine_build.sh
- sudo zip -j dist/lbrynet-windows.zip dist/lbrynet.exe
deploy:
provider: releases
@@ -125,7 +127,7 @@ jobs:
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server .
- travis_retry docker build -t lbry/wallet-server:$TRAVIS_TAG -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:$TRAVIS_TAG
- if: tag IS blank AND branch = master AND NOT type IN (pull_request)
stage: build
@@ -133,7 +135,7 @@ jobs:
script:
- set -e
- echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server .
- travis_retry docker build -t lbry/wallet-server:master -f scripts/Dockerfile.wallet_server --build-arg TORBA_VERSION=${TORBA} .
- docker push lbry/wallet-server:master
cache:
+1 -1
View File
@@ -41,7 +41,7 @@ This project is MIT licensed. For the full license, see [LICENSE](LICENSE).
## Security
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.io/faq/gpg-key) if you need it.
We take security seriously. Please contact security@lbry.com regarding any security issues. [Our GPG key is here](https://lbry.com/faq/gpg-key) if you need it.
## Contact
+1 -1
View File
@@ -1,4 +1,4 @@
__name__ = "lbrynet"
__version__ = "0.38.0rc5"
__version__ = "0.38.0rc6"
version = tuple(__version__.split('.'))
+19 -12
View File
@@ -121,7 +121,7 @@ class Toggle(Setting[bool]):
)
parser.add_argument(
self.no_cli_name,
help=f"Opposite of --{self.cli_name}",
help=f"Opposite of {self.cli_name}",
dest=self.name,
action="store_false",
default=NOT_SET
@@ -192,7 +192,21 @@ class MaxKeyFee(Setting[dict]):
)
class Servers(Setting[list]):
class ListSetting(Setting[list]):
def validate(self, val):
assert isinstance(val, (tuple, list)), \
f"Setting '{self.name}' must be a tuple or list."
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
help=self.doc,
action='append'
)
class Servers(ListSetting):
def validate(self, val):
assert isinstance(val, (tuple, list)), \
@@ -225,15 +239,8 @@ class Servers(Setting[list]):
return [f"{host}:{port}" for host, port in value]
return value
def contribute_to_argparse(self, parser: ArgumentParser):
parser.add_argument(
self.cli_name,
help=self.doc,
action='append'
)
class Strings(Setting[list]):
class Strings(ListSetting):
def validate(self, val):
assert isinstance(val, (tuple, list)), \
@@ -268,7 +275,7 @@ class ArgumentAccess:
def load(self, args):
for setting in self.configuration.get_settings():
value = getattr(args, setting.name, NOT_SET)
if value != NOT_SET and not (isinstance(setting, Servers) and value is None):
if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None):
self.args[setting.name] = setting.deserialize(value)
def __contains__(self, item: str):
@@ -519,7 +526,7 @@ class Config(CLIConfig):
# servers
reflector_servers = Servers("Reflector re-hosting servers", [
('reflector.lbry.io', 5566)
('reflector.lbry.com', 5566)
])
lbryum_servers = Servers("SPV wallet servers", [
('lbryumx1.lbry.com', 50001), # US EAST
+4 -2
View File
@@ -169,7 +169,7 @@ def add_command_parser(parent, command):
def get_argument_parser():
main = ArgumentParser(
'lbrynet', description='An interface to the LBRY Network.'
'lbrynet', description='An interface to the LBRY Network.', allow_abbrev=False,
)
main.add_argument(
'-v', '--version', dest='cli_version', action="store_true",
@@ -192,8 +192,8 @@ def get_argument_parser():
help=('Enable debug output. Optionally specify loggers for which debug output '
'should selectively be applied.')
)
start.set_defaults(command='start', start_parser=start)
Config.contribute_to_argparse(start)
start.set_defaults(command='start', start_parser=start, doc=start.format_help())
api = Daemon.get_api_definitions()
groups = {}
@@ -239,6 +239,7 @@ def main(argv=None):
if args.help:
args.start_parser.print_help()
return 0
default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
file_handler = logging.handlers.RotatingFileHandler(
conf.log_file_path, maxBytes=2097152, backupCount=5
@@ -264,6 +265,7 @@ def main(argv=None):
loop.set_debug(True)
else:
log.setLevel(logging.INFO)
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
+5 -3
View File
@@ -244,13 +244,14 @@ class WalletComponent(Component):
def __init__(self, component_manager):
super().__init__(component_manager)
self.wallet_manager = None
self._started = False
@property
def component(self):
return self.wallet_manager
async def get_status(self):
if self.wallet_manager and self.running:
if self.wallet_manager and self._started:
local_height = self.wallet_manager.ledger.headers.height
remote_height = await self.wallet_manager.ledger.network.get_server_height()
best_hash = self.wallet_manager.get_best_blockhash()
@@ -265,6 +266,7 @@ class WalletComponent(Component):
async def start(self):
log.info("Starting torba wallet")
self.wallet_manager = await LbryWalletManager.from_lbrynet_config(self.conf)
self._started = True
await self.wallet_manager.start()
async def stop(self):
@@ -306,7 +308,7 @@ class BlobComponent(Component):
count = len(self.blob_manager.completed_blob_hashes)
return {
'finished_blobs': count,
'connections': self.blob_manager.connection_manager.status
'connections': {} if not self.blob_manager else self.blob_manager.connection_manager.status
}
@@ -513,7 +515,7 @@ class UPnPComponent(Component):
pass
if external_ip == "0.0.0.0" or not external_ip:
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
log.warning("unable to get external ip from UPnP, checking lbry.com fallback")
external_ip = await utils.get_external_ip()
if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
+9 -12
View File
@@ -51,7 +51,6 @@ if typing.TYPE_CHECKING:
from lbrynet.wallet.manager import LbryWalletManager
from lbrynet.wallet.ledger import MainNetLedger
from lbrynet.stream.stream_manager import StreamManager
from lbrynet.stream.managed_stream import ManagedStream
log = logging.getLogger(__name__)
@@ -808,16 +807,14 @@ class Daemon(metaclass=JSONRPCServerType):
Returns:
(dict) Dictionary of lbry version information
{
'build': (str) build type (e.g. "dev", "rc", "release"),
'ip': (str) remote ip, if available,
'lbrynet_version': (str) lbrynet_version,
'lbryum_version': (str) lbryum_version,
'lbryschema_version': (str) lbryschema_version,
'os_release': (str) os release string
'os_system': (str) os name
'platform': (str) platform string
'processor': (str) processor type,
'python_version': (str) python version,
'platform': (str) platform string,
'os_release': (str) os release string,
'os_system': (str) os name,
'lbrynet_version': (str) lbrynet version,
'torba_version': (str) torba version,
'build': (str) "dev" | "qa" | "rc" | "release",
}
"""
platform_info = system_info.get_platform()
@@ -1808,9 +1805,9 @@ class Daemon(metaclass=JSONRPCServerType):
Returns: {Paginated[Output]}
"""
if kwargs.pop('valid_channel_signatures', False):
kwargs['is_channel_signature_valid'] = 1
elif kwargs.pop('invalid_channel_signatures', False):
kwargs['is_channel_signature_valid'] = 0
kwargs['signature_valid'] = 1
if kwargs.pop('invalid_channel_signatures', False):
kwargs['signature_valid'] = 0
page_num, page_size = abs(kwargs.pop('page', 1)), min(abs(kwargs.pop('page_size', 10)), 50)
kwargs.update({'offset': page_size * (page_num-1), 'limit': page_size})
txos, offset, total = await self.ledger.claim_search(**kwargs)
@@ -123,8 +123,8 @@ class LBRYioFeed(MarketFeed):
def __init__(self):
super().__init__(
"BTCLBC",
"lbry.io",
"https://api.lbry.io/lbc/exchange_rate",
"lbry.com",
"https://api.lbry.com/lbc/exchange_rate",
{},
0.0,
)
@@ -140,8 +140,8 @@ class LBRYioBTCFeed(MarketFeed):
def __init__(self):
super().__init__(
"USDBTC",
"lbry.io",
"https://api.lbry.io/lbc/exchange_rate",
"lbry.com",
"https://api.lbry.com/lbc/exchange_rate",
{},
0.0,
)
@@ -184,12 +184,13 @@ class JSONResponseEncoder(JSONEncoder):
output['value_type'] = txo.claim.claim_type
if self.include_protobuf:
output['protobuf'] = hexlify(txo.claim.to_bytes())
if txo.channel is not None:
output['signing_channel'] = self.encode_output(txo.channel)
if check_signature and txo.claim.is_signed:
if check_signature and txo.claim.is_signed:
if txo.channel is not None:
output['signing_channel'] = self.encode_output(txo.channel)
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
else:
output['signing_channel'] = {'channel_id': txo.claim.signing_channel_id}
output['is_channel_signature_valid'] = False
if txo.channel:
output['is_channel_signature_valid'] = txo.is_signed_by(txo.channel, self.ledger)
except DecodeError:
pass
return output
+2
View File
@@ -3,6 +3,7 @@ import os
import logging.handlers
from lbrynet import build_type, __version__ as lbrynet_version
from torba import __version__ as torba_version
log = logging.getLogger(__name__)
@@ -18,6 +19,7 @@ def get_platform() -> dict:
"os_release": platform.release(),
"os_system": os_system,
"lbrynet_version": lbrynet_version,
"torba_version": torba_version,
"build": build_type.BUILD, # CI server sets this during build step
}
if p["os_system"] == "Linux":
+2 -3
View File
@@ -195,14 +195,13 @@ class Stream(BaseClaim):
claim['source']['hash'] = self.source.file_hash
if 'sd_hash' in claim['source']:
claim['source']['sd_hash'] = self.source.sd_hash
if 'media_type' in claim['source']:
claim['stream_type'] = guess_stream_type(claim['source']['media_type'])
fee = claim.get('fee', {})
if 'address' in fee:
fee['address'] = self.fee.address
if 'amount' in fee:
fee['amount'] = str(self.fee.amount)
stream_type = self.message.WhichOneof('type')
if stream_type:
claim['stream_type'] = stream_type
return claim
def update(self, file_path=None, height=None, width=None, duration=None, **kwargs):
+1 -1
View File
@@ -4,7 +4,7 @@ from typing import NamedTuple, Tuple
def _create_url_regex():
# see https://spec.lbry.io/
# see https://spec.lbry.com/
def _named(name, regex):
return "(?P<" + name + ">" + regex + ")"
+3 -3
View File
@@ -75,7 +75,7 @@ def obfuscate(plain):
return rot13(base64.b64encode(plain).decode())
def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
def check_connection(server="lbry.com", port=80, timeout=5) -> bool:
"""Attempts to open a socket to server:port and returns True if successful."""
log.debug('Checking connection to %s:%s', server, port)
try:
@@ -102,7 +102,7 @@ def check_connection(server="lbry.io", port=80, timeout=5) -> bool:
return False
async def async_check_connection(server="lbry.io", port=80, timeout=5) -> bool:
async def async_check_connection(server="lbry.com", port=80, timeout=5) -> bool:
return await asyncio.get_event_loop().run_in_executor(None, check_connection, server, port, timeout)
@@ -271,7 +271,7 @@ async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[a
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
try:
async with aiohttp_request("get", "https://api.lbry.io/ip") as resp:
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
response = await resp.json()
if response['success']:
return response['data']['ip']
+1 -1
View File
@@ -59,7 +59,7 @@ class WalletDatabase(BaseDatabase):
channel_ids = set()
for txo in txos:
if txo.script.is_claim_name or txo.script.is_update_claim:
if txo.is_claim and txo.can_decode_claim:
if txo.claim.is_signed:
channel_ids.add(txo.claim.signing_channel_id)
if txo.claim.is_channel and my_account is not None:
+84 -27
View File
@@ -116,7 +116,7 @@ class SQLDB:
channel_join integer, -- height at which claim got valid signature / joined channel
signature bytes,
signature_digest bytes,
is_channel_signature_valid bool not null default false,
signature_valid bool,
effective_amount integer not null default 0,
support_amount integer not null default 0,
@@ -126,21 +126,26 @@ class SQLDB:
trending_global integer not null default 0
);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_claims_in_channel_idx on claim (signature_valid, channel_hash, normalized);
create index if not exists claim_id_idx on claim (claim_id);
create index if not exists claim_normalized_idx on claim (normalized);
create index if not exists claim_resolve_idx on claim (normalized, claim_id);
create index if not exists claim_txo_hash_idx on claim (txo_hash);
create index if not exists claim_channel_hash_idx on claim (channel_hash);
create index if not exists claim_release_time_idx on claim (release_time);
create index if not exists claim_timestamp_idx on claim (timestamp);
create index if not exists claim_height_idx on claim (height);
create index if not exists claim_activation_height_idx on claim (activation_height);
create index if not exists claim_expiration_height_idx on claim (expiration_height);
create index if not exists claim_public_key_hash_idx on claim (public_key_hash);
create index if not exists claim_claim_type_idx on claim (claim_type);
create index if not exists claim_stream_type_idx on claim (stream_type);
create index if not exists claim_media_type_idx on claim (media_type);
create index if not exists claim_signature_valid_idx on claim (signature_valid);
create index if not exists claim_effective_amount_idx on claim (effective_amount);
create index if not exists claim_trending_group_idx on claim (trending_group);
create index if not exists claim_trending_mixed_idx on claim (trending_mixed);
@@ -314,7 +319,7 @@ class SQLDB:
:claim_type, :media_type, :stream_type, :timestamp, :timestamp, :height, :height,
CASE WHEN :release_time IS NOT NULL THEN :release_time ELSE :timestamp END,
CASE WHEN :normalized NOT IN (SELECT normalized FROM claimtrie) THEN :height END,
CASE WHEN :height >= 262974 THEN :height+2102400 ELSE :height+262974 END,
CASE WHEN :height >= 137181 THEN :height+2102400 ELSE :height+262974 END,
:claim_name||COALESCE(
(SELECT shortest_id(claim_id, :claim_id) FROM claim WHERE normalized = :normalized),
'#'||substr(:claim_id, 1, 1)
@@ -384,10 +389,12 @@ class SQLDB:
'support', {'txo_hash__in': [sqlite3.Binary(txo_hash) for txo_hash in txo_hashes]}
))
def validate_channel_signatures(self, height, new_claims, updated_claims, spent_claims):
def validate_channel_signatures(self, height, new_claims, updated_claims, spent_claims, timer):
if not new_claims and not updated_claims and not spent_claims:
return
sub_timer = timer.add_timer('segregate channels and signables')
sub_timer.start()
channels, new_channel_keys, signables = {}, {}, {}
for txo in chain(new_claims, updated_claims):
try:
@@ -399,13 +406,19 @@ class SQLDB:
new_channel_keys[txo.claim_hash] = claim.channel.public_key_bytes
else:
signables[txo.claim_hash] = txo
sub_timer.stop()
sub_timer = timer.add_timer('make list of channels we need to lookup')
sub_timer.start()
missing_channel_keys = set()
for txo in signables.values():
claim = txo.claim
if claim.is_signed and claim.signing_channel_hash not in new_channel_keys:
missing_channel_keys.add(claim.signing_channel_hash)
sub_timer.stop()
sub_timer = timer.add_timer('lookup missing channels')
sub_timer.start()
all_channel_keys = {}
if new_channel_keys or missing_channel_keys:
all_channel_keys = dict(self.execute(*query(
@@ -415,7 +428,10 @@ class SQLDB:
set(new_channel_keys) | missing_channel_keys
]
)))
sub_timer.stop()
sub_timer = timer.add_timer('prepare for updating claims')
sub_timer.start()
changed_channel_keys = {}
for claim_hash, new_key in new_channel_keys.items():
if claim_hash not in all_channel_keys or all_channel_keys[claim_hash] != new_key:
@@ -431,16 +447,20 @@ class SQLDB:
'channel_hash': None,
'signature': None,
'signature_digest': None,
'is_channel_signature_valid': False
'signature_valid': None
}
if claim.is_signed:
update.update({
'channel_hash': sqlite3.Binary(claim.signing_channel_hash),
'signature': sqlite3.Binary(txo.get_encoded_signature()),
'signature_digest': sqlite3.Binary(txo.get_signature_digest(self.ledger))
'signature_digest': sqlite3.Binary(txo.get_signature_digest(self.ledger)),
'signature_valid': 0
})
claim_updates.append(update)
sub_timer.stop()
sub_timer = timer.add_timer('find claims affected by a change in channel key')
sub_timer.start()
if changed_channel_keys:
sql = f"""
SELECT * FROM claim WHERE
@@ -454,48 +474,62 @@ class SQLDB:
'channel_hash': sqlite3.Binary(affected_claim['channel_hash']),
'signature': sqlite3.Binary(affected_claim['signature']),
'signature_digest': sqlite3.Binary(affected_claim['signature_digest']),
'is_channel_signature_valid': False
'signature_valid': 0
})
sub_timer.stop()
sub_timer = timer.add_timer('verify signatures')
sub_timer.start()
for update in claim_updates:
channel_pub_key = all_channel_keys.get(update['channel_hash'])
if channel_pub_key and update['signature']:
update['is_channel_signature_valid'] = Output.is_signature_valid(
update['signature_valid'] = Output.is_signature_valid(
bytes(update['signature']), bytes(update['signature_digest']), channel_pub_key
)
sub_timer.stop()
sub_timer = timer.add_timer('update claims')
sub_timer.start()
if claim_updates:
self.db.executemany(f"""
UPDATE claim SET
channel_hash=:channel_hash, signature=:signature, signature_digest=:signature_digest,
is_channel_signature_valid=:is_channel_signature_valid,
signature_valid=:signature_valid,
channel_join=CASE
WHEN is_channel_signature_valid AND :is_channel_signature_valid THEN channel_join
WHEN :is_channel_signature_valid THEN {height}
WHEN signature_valid=1 AND :signature_valid=1 THEN channel_join
WHEN :signature_valid=1 THEN {height}
END,
canonical_url=CASE
WHEN is_channel_signature_valid AND :is_channel_signature_valid THEN canonical_url
WHEN :is_channel_signature_valid THEN
WHEN signature_valid=1 AND :signature_valid=1 THEN canonical_url
WHEN :signature_valid=1 THEN
(SELECT short_url FROM claim WHERE claim_hash=:channel_hash)||'/'||
claim_name||COALESCE(
(SELECT shortest_id(other_claim.claim_id, claim.claim_id) FROM claim AS other_claim
WHERE other_claim.normalized = claim.normalized AND
WHERE other_claim.signature_valid = 1 AND
other_claim.channel_hash = :channel_hash AND
other_claim.is_channel_signature_valid = 1),
other_claim.normalized = claim.normalized),
'#'||substr(claim_id, 1, 1)
)
END
WHERE claim_hash=:claim_hash;
""", claim_updates)
sub_timer.stop()
sub_timer = timer.add_timer('update claims affected by spent channels')
sub_timer.start()
if spent_claims:
self.execute(
f"""
UPDATE claim SET is_channel_signature_valid=0, channel_join=NULL, canonical_url=NULL
UPDATE claim SET
signature_valid=CASE WHEN signature IS NOT NULL THEN 0 END,
channel_join=NULL, canonical_url=NULL
WHERE channel_hash IN ({','.join('?' for _ in spent_claims)})
""", [sqlite3.Binary(cid) for cid in spent_claims]
)
sub_timer.stop()
sub_timer = timer.add_timer('update channels')
sub_timer.start()
if channels:
self.db.executemany(
"""
@@ -510,17 +544,21 @@ class SQLDB:
self.ledger.public_key_to_address(txo.claim.channel.public_key_bytes)))
} for claim_hash, txo in channels.items()]
)
sub_timer.stop()
sub_timer = timer.add_timer('update claims_in_channel counts')
sub_timer.start()
if all_channel_keys:
self.db.executemany(f"""
UPDATE claim SET
claims_in_channel=(
SELECT COUNT(*) FROM claim AS claim_in_channel
WHERE claim_in_channel.channel_hash=claim.claim_hash AND
claim_in_channel.is_channel_signature_valid
WHERE claim_in_channel.signature_valid=1 AND
claim_in_channel.channel_hash=claim.claim_hash
)
WHERE claim_hash = ?
""", [(sqlite3.Binary(channel_hash),) for channel_hash in all_channel_keys.keys()])
sub_timer.stop()
def _update_support_amount(self, claim_hashes):
if claim_hashes:
@@ -617,6 +655,11 @@ class SQLDB:
r(self._update_effective_amount, height)
r(self._perform_overtake, height, [], [])
def get_expiring(self, height):
return self.execute(
f"SELECT claim_hash, normalized FROM claim WHERE expiration_height = {height}"
)
def advance_txs(self, height, all_txs, header, daemon_height, timer):
insert_claims = []
update_claims = []
@@ -637,8 +680,8 @@ class SQLDB:
)
body_timer.start()
delete_claim_hashes.update({r['claim_hash'] for r in spent_claims})
delete_support_txo_hashes.update({r['txo_hash'] for r in spent_supports})
deleted_claim_names.update({r['normalized'] for r in spent_claims})
delete_support_txo_hashes.update({r['txo_hash'] for r in spent_supports})
recalculate_claim_hashes.update({r['claim_hash'] for r in spent_supports})
delete_others.update(spent_others)
# Outputs
@@ -656,18 +699,28 @@ class SQLDB:
delete_claim_hashes.discard(claim_hash)
delete_others.discard(output.ref.hash) # claim insertion and update occurring in the same block
body_timer.stop()
skip_claim_timer = timer.add_timer('skip insertion of abandoned claims')
skip_claim_timer.start()
for new_claim in list(insert_claims):
if new_claim.ref.hash in delete_others:
insert_claims.remove(new_claim)
skip_claim_timer.stop()
expire_timer = timer.add_timer('recording expired claims')
expire_timer.start()
for expired in self.get_expiring(height):
delete_claim_hashes.add(expired['claim_hash'])
deleted_claim_names.add(expired['normalized'])
expire_timer.stop()
r = timer.run
r(self.delete_claims, delete_claim_hashes)
r(self.delete_supports, delete_support_txo_hashes)
r(self.insert_claims, insert_claims, header)
r(self.update_claims, update_claims, header)
r(self.validate_channel_signatures, height, insert_claims, update_claims, delete_claim_hashes)
r(self.validate_channel_signatures, height, insert_claims,
update_claims, delete_claim_hashes, forward_timer=True)
r(self.insert_supports, insert_supports)
r(self.update_claimtrie, height, recalculate_claim_hashes, deleted_claim_names, forward_timer=True)
r(calculate_trending, self.db, height, self.main.first_sync, daemon_height)
@@ -752,11 +805,15 @@ class SQLDB:
if 'claim_type' in constraints:
constraints['claim.claim_type'] = CLAIM_TYPES[constraints.pop('claim_type')]
if 'stream_types' in constraints:
constraints['claim.stream_type__in'] = [
STREAM_TYPES[stream_type] for stream_type in constraints.pop('stream_types')
]
stream_types = constraints.pop('stream_types')
if stream_types:
constraints['claim.stream_type__in'] = [
STREAM_TYPES[stream_type] for stream_type in stream_types
]
if 'media_types' in constraints:
constraints['claim.media_type__in'] = constraints.pop('media_types')
media_types = constraints.pop('media_types')
if media_types:
constraints['claim.media_type__in'] = media_types
_apply_constraints_for_array_attributes(constraints, 'tag')
_apply_constraints_for_array_attributes(constraints, 'language')
@@ -796,14 +853,14 @@ class SQLDB:
claim.trending_local, claim.trending_global,
claim.short_url, claim.canonical_url,
claim.channel_hash, channel.txo_hash AS channel_txo_hash,
channel.height AS channel_height, claim.is_channel_signature_valid
channel.height AS channel_height, claim.signature_valid
""", **constraints
)
INTEGER_PARAMS = {
'height', 'creation_height', 'activation_height', 'expiration_height',
'timestamp', 'creation_timestamp', 'release_time',
'tx_position', 'channel_join', 'is_channel_signature_valid',
'tx_position', 'channel_join', 'signature_valid',
'amount', 'effective_amount', 'support_amount',
'trending_group', 'trending_mixed',
'trending_local', 'trending_global',
@@ -866,7 +923,7 @@ class SQLDB:
else:
query['order_by'] = ['^channel_join']
query['channel_hash'] = channel['claim_hash']
query['is_channel_signature_valid'] = 1
query['signature_valid'] = 1
elif set(query) == {'name'}:
query['is_controlling'] = 1
matches = self._search(**query, limit=1)
+7
View File
@@ -87,6 +87,13 @@ class Output(BaseOutput):
return self.script.values['claim']
raise ValueError('Only claim name and claim update have the claim payload.')
@property
def can_decode_claim(self):
try:
return self.claim
except:
return False
@property
def permanent_url(self) -> str:
if self.script.is_claim_involved:
+1 -1
View File
@@ -24,6 +24,6 @@ theme:
- type: 'telegram'
link: 'https://t.me/lbryofficial'
- type: 'discord'
link: 'https://chat.lbry.io'
link: 'https://chat.lbry.com'
- type: 'reddit'
link: 'https://reddit.com/r/lbry'
+3 -1
View File
@@ -1,5 +1,7 @@
FROM debian:buster-slim
ARG TORBA_VERSION=master
RUN apt-get update && \
apt-get upgrade -y && \
apt-get install -y --no-install-recommends \
@@ -15,7 +17,7 @@ RUN python3.7 -m pip install --upgrade pip setuptools wheel
COPY . /app
WORKDIR /app
RUN python3.7 -m pip install git+https://github.com/lbryio/torba.git#egg=torba
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git@${TORBA_VERSION}#egg=torba
RUN python3.7 -m pip install -e .
# Orchstr8 API
+2 -1
View File
@@ -1,6 +1,7 @@
FROM debian:buster-slim
ARG user=lbry
ARG TORBA_VERSION=master
# create an unprivileged user
RUN groupadd -g 999 $user && useradd -r -u 999 -g $user $user
@@ -23,7 +24,7 @@ WORKDIR /home/$user
RUN python3.7 -m pip install --upgrade pip setuptools
# get torba
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git#egg=torba
RUN python3.7 -m pip install --user git+https://github.com/lbryio/torba.git@${TORBA_VERSION}#egg=torba
# get uvloop
RUN python3.7 -m pip install --user uvloop
+1 -1
View File
@@ -47,7 +47,7 @@ async def main(blob_hash: str, url: str):
if __name__ == "__main__": # usage: python download_blob_from_peer.py <blob_hash> [host url:port]
url = 'reflector.lbry.io:5567'
url = 'reflector.lbry.com:5567'
if len(sys.argv) > 2:
url = sys.argv[2]
asyncio.run(main(sys.argv[1], url))
+1 -1
View File
@@ -26,7 +26,7 @@ def extract_uris(response):
async def get_frontpage_uris():
session = aiohttp.ClientSession()
try:
response = await session.get("https://api.lbry.io/file/list_homepage", timeout=10.0)
response = await session.get("https://api.lbry.com/file/list_homepage", timeout=10.0)
if response.status != 200:
print("API returned non 200 code!!")
return
+9 -2
View File
@@ -1,9 +1,11 @@
set -x
TORBA=$1
# must also be updated in travis.yml
TORBA_VERSION=master
rm -rf /tmp/.wine-*
apt-get -qq update
apt-get -qq install -y git
pip install setuptools_scm
cd lbry
@@ -12,8 +14,13 @@ cd lbry
wget -Onetifaces-0.10.7-cp37-cp37m-win32.whl https://ci.appveyor.com/api/buildjobs/6hworunifsymrhp2/artifacts/dist%2Fnetifaces-0.10.7-cp37-cp37m-win32.whl
pip install netifaces-0.10.7-cp37-cp37m-win32.whl
pip install git+https://github.com/lbryio/torba.git@${TORBA}#egg=torba
git clone --depth=1 --single-branch --branch ${TORBA_VERSION} https://github.com/lbryio/torba.git
cd torba
pip install .
cd ..
rm -rf torba
pip show torba
pip install -e .
pip install pywin32
+3 -3
View File
@@ -9,8 +9,8 @@ setup(
name=__name__,
version=__version__,
author="LBRY Inc.",
author_email="hello@lbry.io",
url="https://lbry.io",
author_email="hello@lbry.com",
url="https://lbry.com",
description="A decentralized media library and marketplace",
long_description=open(README_PATH, encoding='utf-8').read(),
long_description_content_type="text/markdown",
@@ -25,7 +25,7 @@ setup(
install_requires=[
'torba==0.5.5',
'aiohttp==3.5.4',
'aioupnp',
'aioupnp==0.0.12',
'appdirs==1.4.3',
'certifi>=2018.11.29',
'colorama==0.3.7',
+16 -1
View File
@@ -141,8 +141,20 @@ class ClaimSearchCommand(ClaimTestCase):
await self.channel_abandon(claim_id=self.channel_id)
await self.assertFindsClaims([], channel_ids=[self.channel_id], valid_channel_signatures=True)
await self.assertFindsClaims([signed2], channel_ids=[channel_id2], valid_channel_signatures=True)
# pass `invalid_channel_signatures=False` to catch a bug in argument processing
await self.assertFindsClaims([signed2], channel_ids=[channel_id2, self.channel_id],
valid_channel_signatures=True)
valid_channel_signatures=True, invalid_channel_signatures=False)
# invalid signature still returns channel_id
self.ledger._tx_cache.clear()
invalid_claims = await self.claim_search(invalid_channel_signatures=True)
self.assertEqual(3, len(invalid_claims))
self.assertTrue(all([not c['is_channel_signature_valid'] for c in invalid_claims]))
self.assertEqual({'channel_id': self.channel_id}, invalid_claims[0]['signing_channel'])
valid_claims = await self.claim_search(valid_channel_signatures=True)
self.assertEqual(1, len(valid_claims))
self.assertTrue(all([c['is_channel_signature_valid'] for c in valid_claims]))
self.assertEqual('@abc', valid_claims[0]['signing_channel']['name'])
# abandoned stream won't show up for streams in channel search
await self.stream_abandon(txid=signed2['txid'], nout=0)
@@ -616,6 +628,7 @@ class StreamCommands(ClaimTestCase):
}]
fixed_values['thumbnail'] = {'url': fixed_values.pop('thumbnail_url')}
fixed_values['release_time'] = str(values['release_time'])
fixed_values['stream_type'] = 'binary'
fixed_values['source'] = {
'hash': '56bf5dbae43f77a63d075b0f2ae9c7c3e3098db93779c7f9840da0f4db9c2f8c8454f4edd1373e2b64ee2e68350d916e',
'media_type': 'application/octet-stream',
@@ -639,6 +652,7 @@ class StreamCommands(ClaimTestCase):
stream = tx['outputs'][0]['value']
self.assertEqual(
stream, {
'stream_type': 'binary',
'source': {
'size': '3',
'media_type': 'application/octet-stream',
@@ -657,6 +671,7 @@ class StreamCommands(ClaimTestCase):
fixed_values['source']['sd_hash'] = stream['source']['sd_hash']
self.assertEqual(
stream, {
'stream_type': 'binary',
'source': {
'size': '3',
'media_type': 'application/octet-stream',
+7 -5
View File
@@ -134,7 +134,6 @@ class ResolveCommand(CommandTestCase):
channel_id = self.get_claim_id(
await self.channel_create('@abc', '1.1', allow_duplicate_name=True))
await self.assertResolvesToClaimId(f'@abc', channel_id)
await self.assertResolvesToClaimId(f'@abc#{channel_id[0]}', channel_id)
await self.assertResolvesToClaimId(f'@abc#{channel_id[:10]}', channel_id)
await self.assertResolvesToClaimId(f'@abc#{channel_id}', channel_id)
channel = (await self.claim_search(claim_id=channel_id))[0]
@@ -168,6 +167,7 @@ class ResolveCommand(CommandTestCase):
async def test_abandoned_channel_with_signed_claims(self):
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
orphan_claim = await self.stream_create('on-channel-claim', '0.0001', channel_id=channel['claim_id'])
abandoned_channel_id = channel['claim_id']
await self.channel_abandon(txid=channel['txid'], nout=0)
channel = (await self.channel_create('@abc', '1.0'))['outputs'][0]
orphan_claim_id = orphan_claim['outputs'][0]['claim_id']
@@ -178,11 +178,13 @@ class ResolveCommand(CommandTestCase):
self.assertEqual(response, {
'lbry://@abc/on-channel-claim': {'error': 'lbry://@abc/on-channel-claim did not resolve to a claim'}
})
response = await self.resolve('lbry://on-channel-claim')
self.assertNotIn('is_channel_signature_valid', response['lbry://on-channel-claim'])
response = (await self.resolve('lbry://on-channel-claim'))['lbry://on-channel-claim']
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
direct_uri = 'lbry://on-channel-claim#' + orphan_claim_id
response = await self.resolve(direct_uri)
self.assertNotIn('is_channel_signature_valid', response[direct_uri])
response = (await self.resolve(direct_uri))[direct_uri]
self.assertFalse(response['is_channel_signature_valid'])
self.assertEqual({'channel_id': abandoned_channel_id}, response['signing_channel'])
await self.stream_abandon(claim_id=orphan_claim_id)
uri = 'lbry://@abc/on-channel-claim'
+7 -1
View File
@@ -4,7 +4,7 @@ import types
import tempfile
import unittest
import argparse
from lbrynet.conf import Config, BaseConfig, String, Integer, Toggle, Servers, NOT_SET
from lbrynet.conf import Config, BaseConfig, String, Integer, Toggle, Servers, Strings, NOT_SET
from lbrynet.error import InvalidCurrencyError
@@ -14,6 +14,7 @@ class TestConfig(BaseConfig):
test_false_toggle = Toggle('toggle help', False)
test_true_toggle = Toggle('toggle help', True)
servers = Servers('servers help', [('localhost', 80)])
strings = Strings('cheese', ['string'])
class ConfigurationTests(unittest.TestCase):
@@ -54,6 +55,7 @@ class ConfigurationTests(unittest.TestCase):
self.assertTrue(c.test_true_toggle)
self.assertFalse(c.test_false_toggle)
self.assertEqual(c.servers, [('localhost', 80)])
self.assertEqual(c.strings, ['string'])
args = parser.parse_args(['--test-str', 'blah'])
c = TestConfig.create_from_arguments(args)
@@ -80,6 +82,10 @@ class ConfigurationTests(unittest.TestCase):
c = TestConfig.create_from_arguments(args)
self.assertEqual(c.servers, [('localhost', 1), ('192.168.0.1', 2)])
args = parser.parse_args(['--strings=cheddar', '--strings=mozzarella'])
c = TestConfig.create_from_arguments(args)
self.assertEqual(c.strings, ['cheddar', 'mozzarella'])
def test_environment(self):
c = TestConfig()
self.assertEqual(c.test_str, 'the default')
+23 -1
View File
@@ -140,7 +140,7 @@ class TestSQLDB(unittest.TestCase):
return [otx[0].tx.outputs[0] for otx in txs]
def state(self, controlling=None, active=None, accepted=None):
self.assertEqual(controlling or [], self.get_controlling())
self.assertEqual(controlling, self.get_controlling())
self.assertEqual(active or [], self.get_active())
self.assertEqual(accepted or [], self.get_accepted())
@@ -288,6 +288,28 @@ class TestSQLDB(unittest.TestCase):
accepted=[]
)
def test_winning_claim_expires_and_another_takes_over(self):
advance, state = self.advance, self.state
advance(10, [self.get_stream('Claim A', 11*COIN)])
advance(20, [self.get_stream('Claim B', 10*COIN)])
state(
controlling=('Claim A', 11*COIN, 11*COIN, 10),
active=[('Claim B', 10*COIN, 10*COIN, 20)],
accepted=[]
)
advance(262984, [])
state(
controlling=('Claim B', 10*COIN, 10*COIN, 20),
active=[],
accepted=[]
)
advance(262994, [])
state(
controlling=None,
active=[],
accepted=[]
)
def test_trending(self):
advance, state = self.advance, self.state
no_trend = self.get_stream('Claim A', COIN)